1577514423

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/JohnSnowLabs/spark-nlp/commit/<a class=hub.com/JohnSnowLabs/spark-nlp/commit/61ef0a3bec56138e2b02625a512fa0ad8e5b347a">61ef0a3be<a href="https://github.com/JohnSnowLabs/spark-nlp/commit/61ef0a3bec56138e2b02625a512fa0ad8e5b347a">">Merge </a><a class="double-link" href="https://github.com/JohnSnowLabs/spark-nlp/commit/<a class="double-link" href="https://github.com/JohnSnowLabs/spark-nlp/commit/7038e766aea858dde9d49a1c4c9bd518c38c7d8a">7038e766a</a>">7038e766a</a><a href="https://github.com/JohnSnowLabs/spark-nlp/commit/61ef0a3bec56138e2b02625a512fa0ad8e5b347a"> into 3baf344ba">3baf344ba</a>

Pull Request Pull Request #6546: Release/340 release candidate

Run Details

1356 of 1356 new or added lines in 68 files covered. (100.0%)

7939 of 16034 relevant lines covered (49.51%)

0.5 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/scala/com/johnsnowlabs/ml/tensorflow/TensorflowSpell.scala

/*
 * Copyright 2017-2021 John Snow Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.johnsnowlabs.ml.tensorflow

import com.johnsnowlabs.ml.tensorflow.TensorResources._
import com.johnsnowlabs.nlp.annotators.ner.Verbose
import com.johnsnowlabs.nlp.annotators.spell.context.LangModelSentence
import org.tensorflow.Graph

import scala.collection.JavaConversions.asScalaBuffer


class TensorflowSpell(
                       val tensorflow: TensorflowWrapper,
                       val verboseLevel: Verbose.Value
                     ) extends Logging with Serializable {


  val lossKey = "Add:0"
  val dropoutRate = "dropout_rate"

  // these are the inputs to the graph
  val wordIds = "input_batch"
  val contextIds = "output_batch_cids"
  val contextWordIds = "output_batch_wids"
  val inputLens = "input_batch_lengths"

  // for fast evaluation
  val testWids = "test_wids"
  val testCids = "test_cids"
  val losses = "test_losses"

  // training stuff
  val globalStepKey = "train/global_step"
  val initialLearningRateKey = "train/initial_learning_rate"
  val finalLearningRateKey = "train/final_learning_rate"
  val updatesKey = "train/updates"

  // Controls the frequency at which we report train progress, can be disabled completely by changing the log level
  val checkPointStep = 200
  val learningRate = "train/learning_rate/Merge:0"
  val initKey = "init"

  /* returns the loss associated with the last word, given previous history  */
  def predict(dataset: Array[Array[Int]], cids: Array[Array[Int]], cwids: Array[Array[Int]], configProtoBytes: Option[Array[Byte]] = None): Iterator[Float] = {

    val tensors = new TensorResources

    val lossWords = tensorflow.getTFSession(configProtoBytes = configProtoBytes).runner
      .feed(dropoutRate, tensors.createTensor(1.0f))
      .feed(wordIds, tensors.createTensor(dataset.map(_.dropRight(1))))
      .feed(contextIds, tensors.createTensor(cids.map(_.tail)))
      .feed(contextWordIds, tensors.createTensor(cwids.map(_.tail)))
      .fetch(lossKey)
      .run()

    tensors.clearTensors()

    val result = extractFloats(lossWords.get(0))
    val width = dataset.head.length
    result.grouped(width - 1).map(_.last)
  }


  /* returns the loss associated with the last word, given previous history  */
  def pplEachWord(dataset: Array[Array[Int]], cids: Array[Array[Int]], cwids: Array[Array[Int]], configProtoBytes: Option[Array[Byte]] = None): Array[Float] = {

    val tensors = new TensorResources

    val lossWords = tensorflow.getTFSession(configProtoBytes = configProtoBytes).runner
      .feed(dropoutRate, tensors.createTensor(1.0f))
      .feed(wordIds, tensors.createTensor(dataset.map(_.dropRight(1))))
      .feed(contextIds, tensors.createTensor(cids.map(_.tail)))
      .feed(contextWordIds, tensors.createTensor(cwids.map(_.tail)))
      .feed(inputLens, tensors.createTensor(dataset.map(_.length)))
      .fetch(lossKey)
      .run()

    tensors.clearTensors()
    extractFloats(lossWords.get(0))
  }


  def predict_(dataset: Array[Array[Int]], cids: Array[Array[Int]], cwids: Array[Array[Int]],
               candCids: Array[Int], candWids: Array[Int],
               configProtoBytes: Option[Array[Byte]] = None) = {

    val tensors = new TensorResources
    val paths = (dataset, cids, cwids).zipped.toList

    paths.flatMap { case (pathIds, pathCids, pathWids) =>
      val lossWords = tensorflow.getTFSession(configProtoBytes = configProtoBytes).runner
        .feed(dropoutRate, tensors.createTensor(1.0f))
        .feed(wordIds, tensors.createTensor(Array(pathIds)))
        .feed(contextIds, tensors.createTensor(Array(pathCids.tail)))
        .feed(contextWordIds, tensors.createTensor(Array(pathWids.tail)))
        .feed(testCids, tensors.createTensor(Array(candCids)))
        .feed(testWids, tensors.createTensor(Array(candWids)))
        .feed(inputLens, tensors.createTensor(Array(pathIds.length)))
        .fetch(losses)
        .run()

      tensors.clearTensors()
      val r = extractFloats(lossWords.get(0))
      r
    }
  }

  def train(train: => Iterator[Array[LangModelSentence]], valid: => Iterator[Array[LangModelSentence]],
            epochs: Int, batchSize: Int, initialRate: Float, finalRate: Float): Unit = {

    val graph = new Graph()
    val config = Array[Byte](50, 2, 32, 1, 56, 1)
    val session = tensorflow.createSession(Some(config))
    session.runner.addTarget(initKey).run()

    var bestScore = Double.MaxValue
    for (epoch <- 0 until epochs) {
      logger.info(s"Training language model: epoch $epoch")
      for (batch <- train) {
        val tensors = new TensorResources()
        var trainLoss = 0.0
        var trainValidWords = 0

        val tfResponse = session.runner().
          fetch(lossKey).
          fetch(globalStepKey).
          fetch(learningRate).
          fetch(updatesKey).
          feed(dropoutRate, tensors.createTensor(.65f)).
          feed(wordIds, tensors.createTensor(batch.map(_.ids))).
          feed(contextIds, tensors.createTensor(batch.map(_.cids))).
          feed(contextWordIds, tensors.createTensor(batch.map(_.cwids))).
          feed(inputLens, tensors.createTensor(batch.map(_.len))).
          feed(finalLearningRateKey, tensors.createTensor(finalRate)).
          feed(initialLearningRateKey, tensors.createTensor(initialRate)).
          run()

        val loss = tfResponse.lift(0) match {
          case Some(e) => e
          case _ => throw new IllegalArgumentException("Error in TF loss extraction")
        }

        val gs = tfResponse.lift(1) match {
          case Some(e) => e
          case _ => throw new IllegalArgumentException("Error in TF gs extraction")
        }

        val clr = tfResponse.lift(2) match {
          case Some(e) => e
          case _ => throw new IllegalArgumentException("Error in TF clr extraction")
        }

        trainLoss += extractFloats(loss).sum
        val vws = batch.map(_.len).sum
        trainValidWords += vws

        if (extractInt(gs) % checkPointStep == 0) {
          trainLoss /= vws
          val trainPpl = math.exp(trainLoss)
          logger.debug(s"Training Step: ${extractInt(gs)}, LR: ${extractFloats(clr).head}\n Training PPL: $trainPpl")
          trainLoss = 0.0
          trainValidWords = 0
        }
      }

      // The end of one epoch - run validation)
      var devLoss = 0.0
      var devValidWords = 0
      val tensors = new TensorResources()

      for (batch <- valid) {
        val tfValidationResponse = session.runner().
          fetch(lossKey).
          feed(dropoutRate, tensors.createTensor(1.0f)).
          feed(wordIds, tensors.createTensor(batch.map(_.ids))).
          feed(contextIds, tensors.createTensor(batch.map(_.cids))).
          feed(contextWordIds, tensors.createTensor(batch.map(_.cwids))).
          feed(inputLens, tensors.createTensor(batch.map(_.len))).
          run()

        val validLoss = tfValidationResponse.get(0)
        devLoss += extractFloats(validLoss).sum
        devValidWords += batch.map(_.len).sum
      }
      // End of validation
      devLoss /= devValidWords
      val devPpl = math.exp(devLoss)
      logger.debug(s"Validation PPL: $devPpl")
      if (devPpl < bestScore) {
        bestScore = devPpl
      }
    }
  }
}

1	/*
2	* Copyright 2017-2021 John Snow Labs
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16
17	package com.johnsnowlabs.ml.tensorflow
18
19	import com.johnsnowlabs.ml.tensorflow.TensorResources._
20	import com.johnsnowlabs.nlp.annotators.ner.Verbose
21	import com.johnsnowlabs.nlp.annotators.spell.context.LangModelSentence
22	import org.tensorflow.Graph
23
24	import scala.collection.JavaConversions.asScalaBuffer
25
26
27	class TensorflowSpell(
28	val tensorflow: TensorflowWrapper,
29	val verboseLevel: Verbose.Value
30	) extends Logging with Serializable {
31
32
33	val lossKey = "Add:0"	×
34	val dropoutRate = "dropout_rate"	×
35
36	// these are the inputs to the graph
37	val wordIds = "input_batch"	×
38	val contextIds = "output_batch_cids"	×
39	val contextWordIds = "output_batch_wids"	×
40	val inputLens = "input_batch_lengths"	×
41
42	// for fast evaluation
43	val testWids = "test_wids"	×
44	val testCids = "test_cids"	×
45	val losses = "test_losses"	×
46
47	// training stuff
48	val globalStepKey = "train/global_step"	×
49	val initialLearningRateKey = "train/initial_learning_rate"	×
50	val finalLearningRateKey = "train/final_learning_rate"	×
51	val updatesKey = "train/updates"	×
52
53	// Controls the frequency at which we report train progress, can be disabled completely by changing the log level
54	val checkPointStep = 200	×
55	val learningRate = "train/learning_rate/Merge:0"	×
56	val initKey = "init"	×
57
58	/* returns the loss associated with the last word, given previous history */
59	def predict(dataset: Array[Array[Int]], cids: Array[Array[Int]], cwids: Array[Array[Int]], configProtoBytes: Option[Array[Byte]] = None): Iterator[Float] = {
60
61	val tensors = new TensorResources	×
62
63	val lossWords = tensorflow.getTFSession(configProtoBytes = configProtoBytes).runner
64	.feed(dropoutRate, tensors.createTensor(1.0f))
65	.feed(wordIds, tensors.createTensor(dataset.map(_.dropRight(1))))
66	.feed(contextIds, tensors.createTensor(cids.map(_.tail)))
67	.feed(contextWordIds, tensors.createTensor(cwids.map(_.tail)))
68	.fetch(lossKey)
69	.run()	×
70
71	tensors.clearTensors()	×
72
73	val result = extractFloats(lossWords.get(0))	×
74	val width = dataset.head.length	×
75	result.grouped(width - 1).map(_.last)	×
76	}
77
78
79	/* returns the loss associated with the last word, given previous history */
80	def pplEachWord(dataset: Array[Array[Int]], cids: Array[Array[Int]], cwids: Array[Array[Int]], configProtoBytes: Option[Array[Byte]] = None): Array[Float] = {
81
82	val tensors = new TensorResources	×
83
84	val lossWords = tensorflow.getTFSession(configProtoBytes = configProtoBytes).runner
85	.feed(dropoutRate, tensors.createTensor(1.0f))
86	.feed(wordIds, tensors.createTensor(dataset.map(_.dropRight(1))))
87	.feed(contextIds, tensors.createTensor(cids.map(_.tail)))
88	.feed(contextWordIds, tensors.createTensor(cwids.map(_.tail)))
89	.feed(inputLens, tensors.createTensor(dataset.map(_.length)))
90	.fetch(lossKey)
91	.run()	×
92
93	tensors.clearTensors()	×
94	extractFloats(lossWords.get(0))	×
95	}
96
97
98	def predict_(dataset: Array[Array[Int]], cids: Array[Array[Int]], cwids: Array[Array[Int]],
99	candCids: Array[Int], candWids: Array[Int],
100	configProtoBytes: Option[Array[Byte]] = None) = {
101
102	val tensors = new TensorResources	×
103	val paths = (dataset, cids, cwids).zipped.toList	×
104
105	paths.flatMap { case (pathIds, pathCids, pathWids) =>	×
106	val lossWords = tensorflow.getTFSession(configProtoBytes = configProtoBytes).runner
107	.feed(dropoutRate, tensors.createTensor(1.0f))
108	.feed(wordIds, tensors.createTensor(Array(pathIds)))
109	.feed(contextIds, tensors.createTensor(Array(pathCids.tail)))
110	.feed(contextWordIds, tensors.createTensor(Array(pathWids.tail)))
111	.feed(testCids, tensors.createTensor(Array(candCids)))
112	.feed(testWids, tensors.createTensor(Array(candWids)))
113	.feed(inputLens, tensors.createTensor(Array(pathIds.length)))
114	.fetch(losses)
115	.run()	×
116
117	tensors.clearTensors()	×
118	val r = extractFloats(lossWords.get(0))	×
119	r	×
120	}
121	}
122
123	def train(train: => Iterator[Array[LangModelSentence]], valid: => Iterator[Array[LangModelSentence]],
124	epochs: Int, batchSize: Int, initialRate: Float, finalRate: Float): Unit = {
125
126	val graph = new Graph()	×
127	val config = Array[Byte](50, 2, 32, 1, 56, 1)	×
128	val session = tensorflow.createSession(Some(config))	×
129	session.runner.addTarget(initKey).run()	×
130
131	var bestScore = Double.MaxValue	×
132	for (epoch <- 0 until epochs) {	×
133	logger.info(s"Training language model: epoch $epoch")	×
134	for (batch <- train) {	×
135	val tensors = new TensorResources()	×
136	var trainLoss = 0.0	×
137	var trainValidWords = 0	×
138
139	val tfResponse = session.runner().
140	fetch(lossKey).
141	fetch(globalStepKey).
142	fetch(learningRate).
143	fetch(updatesKey).
144	feed(dropoutRate, tensors.createTensor(.65f)).
145	feed(wordIds, tensors.createTensor(batch.map(_.ids))).
146	feed(contextIds, tensors.createTensor(batch.map(_.cids))).
147	feed(contextWordIds, tensors.createTensor(batch.map(_.cwids))).
148	feed(inputLens, tensors.createTensor(batch.map(_.len))).
149	feed(finalLearningRateKey, tensors.createTensor(finalRate)).
150	feed(initialLearningRateKey, tensors.createTensor(initialRate)).
151	run()	×
152
153	val loss = tfResponse.lift(0) match {	×
154	case Some(e) => e
155	case _ => throw new IllegalArgumentException("Error in TF loss extraction")	×
156	}
157
158	val gs = tfResponse.lift(1) match {	×
159	case Some(e) => e
160	case _ => throw new IllegalArgumentException("Error in TF gs extraction")	×
161	}
162
163	val clr = tfResponse.lift(2) match {	×
164	case Some(e) => e
165	case _ => throw new IllegalArgumentException("Error in TF clr extraction")	×
166	}
167
168	trainLoss += extractFloats(loss).sum	×
169	val vws = batch.map(_.len).sum	×
170	trainValidWords += vws	×
171
172	if (extractInt(gs) % checkPointStep == 0) {	×
173	trainLoss /= vws	×
174	val trainPpl = math.exp(trainLoss)	×
175	logger.debug(s"Training Step: ${extractInt(gs)}, LR: ${extractFloats(clr).head}\n Training PPL: $trainPpl")	×
176	trainLoss = 0.0	×
177	trainValidWords = 0	×
178	}
179	}
180
181	// The end of one epoch - run validation)
182	var devLoss = 0.0	×
183	var devValidWords = 0	×
184	val tensors = new TensorResources()	×
185
186	for (batch <- valid) {	×
187	val tfValidationResponse = session.runner().
188	fetch(lossKey).
189	feed(dropoutRate, tensors.createTensor(1.0f)).
190	feed(wordIds, tensors.createTensor(batch.map(_.ids))).
191	feed(contextIds, tensors.createTensor(batch.map(_.cids))).
192	feed(contextWordIds, tensors.createTensor(batch.map(_.cwids))).
193	feed(inputLens, tensors.createTensor(batch.map(_.len))).
194	run()	×
195
196	val validLoss = tfValidationResponse.get(0)	×
197	devLoss += extractFloats(validLoss).sum	×
198	devValidWords += batch.map(_.len).sum	×
199	}
200	// End of validation
201	devLoss /= devValidWords	×
202	val devPpl = math.exp(devLoss)	×
203	logger.debug(s"Validation PPL: $devPpl")	×
204	if (devPpl < bestScore) {	×
205	bestScore = devPpl	×
206	}
207	}
208	}
209	}

JohnSnowLabs / spark-nlp / 1577514423

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous