4992350528

Build Type

github

Committed by GitHub

Commit Message

Merge 424c7ff18 into ef7906c5e

Pull Request Pull Request #13797: SPARKNLP-835: ProtectedParam and ProtectedFeature

Run Details

24 of 24 new or added lines in 6 files covered. (100.0%)

8643 of 13129 relevant lines covered (65.83%)

0.66 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/scala/com/johnsnowlabs/nlp/annotators/audio/Wav2Vec2ForCTC.scala

/*
 * Copyright 2017-2022 John Snow Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.johnsnowlabs.nlp.annotators.audio

import com.johnsnowlabs.ml.ai.Wav2Vec2
import com.johnsnowlabs.ml.tensorflow.{
  ReadTensorflowModel,
  TensorflowWrapper,
  WriteTensorflowModel
}
import com.johnsnowlabs.ml.util.LoadExternalModel.{
  loadJsonStringAsset,
  modelSanityCheck,
  notSupportedEngineError
}
import com.johnsnowlabs.ml.util.ModelEngine
import com.johnsnowlabs.nlp.AnnotatorType.{AUDIO, DOCUMENT}
import com.johnsnowlabs.nlp._
import com.johnsnowlabs.nlp.annotators.audio.feature_extractor.Preprocessor
import com.johnsnowlabs.nlp.serialization.MapFeature
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.ml.param.IntArrayParam
import org.apache.spark.ml.util.Identifiable
import org.apache.spark.sql.SparkSession
import org.json4s._
import org.json4s.jackson.JsonMethods._

/** Wav2Vec2 Model with a language modeling head on top for Connectionist Temporal Classification
  * (CTC). Wav2Vec2 was proposed in wav2vec 2.0: A Framework for Self-Supervised Learning of
  * Speech Representations by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael Auli.
  *
  * The annotator takes audio files and transcribes it as text. The audio needs to be provided
  * pre-processed an array of floats.
  *
  * Note that this annotator is currently not supported on Apple Silicon processors such as the
  * M1/M2 (Apple Silicon). This is due to the processor not supporting instructions for XLA.
  *
  * Pretrained models can be loaded with `pretrained` of the companion object:
  * {{{
  * val speechToText = Wav2Vec2ForCTC.pretrained()
  *   .setInputCols("audio_assembler")
  *   .setOutputCol("text")
  * }}}
  * The default model is `"asr_wav2vec2_base_960h"`, if no name is provided.
  *
  * For available pretrained models please see the [[https://sparknlp.org/models Models Hub]].
  *
  * To see which models are compatible and how to import them see
  * [[https://github.com/JohnSnowLabs/spark-nlp/discussions/5669]] and to see more extended
  * examples, see
  * [[https://github.com/JohnSnowLabs/spark-nlp/blob/master/src/test/scala/com/johnsnowlabs/nlp/annotators/audio/Wav2Vec2ForCTCTestSpec.scala Wav2Vec2ForCTCTestSpec]].
  *
  * ==Example==
  * {{{
  * import spark.implicits._
  * import com.johnsnowlabs.nlp.base._
  * import com.johnsnowlabs.nlp.annotators._
  * import com.johnsnowlabs.nlp.annotators.audio.Wav2Vec2ForCTC
  * import org.apache.spark.ml.Pipeline
  *
  * val audioAssembler: AudioAssembler = new AudioAssembler()
  *   .setInputCol("audio_content")
  *   .setOutputCol("audio_assembler")
  *
  * val speechToText: Wav2Vec2ForCTC = Wav2Vec2ForCTC
  *   .pretrained()
  *   .setInputCols("audio_assembler")
  *   .setOutputCol("text")
  *
  * val pipeline: Pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText))
  *
  * val bufferedSource =
  *   scala.io.Source.fromFile("src/test/resources/audio/csv/audio_floats.csv")
  *
  * val rawFloats = bufferedSource
  *   .getLines()
  *   .map(_.split(",").head.trim.toFloat)
  *   .toArray
  * bufferedSource.close
  *
  * val processedAudioFloats = Seq(rawFloats).toDF("audio_content")
  *
  * val result = pipeline.fit(processedAudioFloats).transform(processedAudioFloats)
  * result.select("text.result").show(truncate = false)
  * +------------------------------------------------------------------------------------------+
  * |result                                                                                    |
  * +------------------------------------------------------------------------------------------+
  * |[MISTER QUILTER IS THE APOSTLE OF THE MIDLE CLASES AND WE ARE GLAD TO WELCOME HIS GOSPEL ]|
  * +------------------------------------------------------------------------------------------+
  * }}}
  * @param uid
  *   required uid for storing annotator to disk
  * @groupname anno Annotator types
  * @groupdesc anno
  *   Required input and expected output annotator types
  * @groupname Ungrouped Members
  * @groupname param Parameters
  * @groupname setParam Parameter setters
  * @groupname getParam Parameter getters
  * @groupname Ungrouped Members
  * @groupprio param  1
  * @groupprio anno  2
  * @groupprio Ungrouped 3
  * @groupprio setParam  4
  * @groupprio getParam  5
  * @groupdesc param
  *   A list of (hyper-)parameter keys this annotator can take. Users can set and get the
  *   parameter values through setters and getters, respectively.
  */
class Wav2Vec2ForCTC(override val uid: String)
    extends AnnotatorModel[Wav2Vec2ForCTC]
    with HasBatchedAnnotateAudio[Wav2Vec2ForCTC]
    with HasAudioFeatureProperties
    with WriteTensorflowModel
    with HasEngine {

  /** Annotator reference id. Used to identify elements in metadata or to refer to this annotator
    * type
    */
  def this() = this(Identifiable.randomUID("Wav2Vec2ForCTC"))

  /** Output annotator type : DOCUMENT
    *
    * @group anno
    */
  override val outputAnnotatorType: AnnotatorType = DOCUMENT

  /** Input annotator type : AUDIO
    *
    * @group anno
    */
  override val inputAnnotatorTypes: Array[AnnotatorType] = Array(AUDIO)

  /** ConfigProto from tensorflow, serialized into byte array. Get with
    * config_proto.SerializeToString()
    *
    * @group param
    */
  val configProtoBytes = new IntArrayParam(
    this,
    "configProtoBytes",
    "ConfigProto from tensorflow, serialized into byte array. Get with config_proto.SerializeToString()")

  /** ConfigProto from tensorflow, serialized into byte array. Get with
    * config_proto.SerializeToString()
    *
    * @group setParam
    */
  def setConfigProtoBytes(bytes: Array[Int]): Wav2Vec2ForCTC.this.type =
    set(this.configProtoBytes, bytes)

  /** ConfigProto from tensorflow, serialized into byte array. Get with
    * config_proto.SerializeToString()
    *
    * @group getParam
    */
  def getConfigProtoBytes: Option[Array[Byte]] =
    get(this.configProtoBytes).map(_.map(_.toByte))

  /** Vocabulary used to encode the words to ids
    *
    * @group param
    */
  val vocabulary: MapFeature[String, BigInt] = new MapFeature(this, "vocabulary").setProtected()

  /** @group setParam */
  def setVocabulary(value: Map[String, BigInt]): this.type = set(vocabulary, value)

  /** It contains TF model signatures for the laded saved model
    *
    * @group param
    */
  val signatures =
    new MapFeature[String, String](model = this, name = "signatures").setProtected()

  /** @group setParam */
  def setSignatures(value: Map[String, String]): this.type = {
    set(signatures, value)
    this
  }

  /** @group getParam */
  def getSignatures: Option[Map[String, String]] = get(this.signatures)

  private var _model: Option[Broadcast[Wav2Vec2]] = None

  /** @group getParam */
  def getModelIfNotSet: Wav2Vec2 = _model.get.value

  /** @group setParam */
  def setModelIfNotSet(spark: SparkSession, tensorflow: TensorflowWrapper): this.type = {
    if (_model.isEmpty) {

      _model = Some(
        spark.sparkContext.broadcast(
          new Wav2Vec2(
            tensorflow,
            configProtoBytes = getConfigProtoBytes,
            vocabs = $$(vocabulary),
            signatures = getSignatures)))
    }
    this
  }

  setDefault(batchSize -> 4)

  /** Takes a document and annotations and produces new annotations of this annotator's annotation
    * type
    *
    * @param batchedAnnotations
    *   Annotations that correspond to inputAnnotationCols generated by previous annotators if any
    * @return
    *   any number of annotations processed for every input annotation. Not necessary one to one
    *   relationship
    */
  override def batchAnnotate(
      batchedAnnotations: Seq[Array[AnnotationAudio]]): Seq[Seq[Annotation]] = {

    // Zip annotations to the row it belongs to
    val audiosWithRow = batchedAnnotations.zipWithIndex
      .flatMap { case (annotations, i) => annotations.map(x => (x, i)) }

    val noneEmptyAudios = audiosWithRow.map(_._1).filter(_.result.nonEmpty).toArray

    val allAnnotations =
      if (noneEmptyAudios.nonEmpty) {
        getModelIfNotSet.predict(
          audios = noneEmptyAudios,
          batchSize = $(batchSize),
          preprocessor = Preprocessor(
            do_normalize = getDoNormalize,
            return_attention_mask = getReturnAttentionMask,
            padding_side = getPaddingSide,
            padding_value = getPaddingValue,
            feature_size = getFeatureSize,
            sampling_rate = getSamplingRate))
      } else {
        Seq.empty[Annotation]
      }

    // Group resulting annotations by rows. If there are not sentences in a given row, return empty sequence
    batchedAnnotations.indices.map(rowIndex => {
      val rowAnnotations = allAnnotations
        // zip each annotation with its corresponding row index
        .zip(audiosWithRow)
        // select the sentences belonging to the current row
        .filter(_._2._2 == rowIndex)
        // leave the annotation only
        .map(_._1)

      if (rowAnnotations.nonEmpty)
        rowAnnotations
      else
        Seq.empty[Annotation]
    })

  }

  override def onWrite(path: String, spark: SparkSession): Unit = {
    super.onWrite(path, spark)
    writeTensorflowModelV2(
      path,
      spark,
      getModelIfNotSet.tensorflowWrapper,
      "_wav_ctc",
      Wav2Vec2ForCTC.tfFile,
      configProtoBytes = getConfigProtoBytes)
  }

}

trait ReadablePretrainedWav2Vec2ForAudioModel
    extends ParamsAndFeaturesReadable[Wav2Vec2ForCTC]
    with HasPretrained[Wav2Vec2ForCTC] {
  override val defaultModelName: Some[String] = Some("asr_wav2vec2_base_960h")

  /** Java compliant-overrides */
  override def pretrained(): Wav2Vec2ForCTC = super.pretrained()

  override def pretrained(name: String): Wav2Vec2ForCTC = super.pretrained(name)

  override def pretrained(name: String, lang: String): Wav2Vec2ForCTC =
    super.pretrained(name, lang)

  override def pretrained(name: String, lang: String, remoteLoc: String): Wav2Vec2ForCTC =
    super.pretrained(name, lang, remoteLoc)
}

trait ReadWav2Vec2ForAudioDLModel extends ReadTensorflowModel {
  this: ParamsAndFeaturesReadable[Wav2Vec2ForCTC] =>

  override val tfFile: String = "wav_ctc_tensorflow"

  def readModel(instance: Wav2Vec2ForCTC, path: String, spark: SparkSession): Unit = {

    val tf = readTensorflowModel(path, spark, "_wav_ctc_tf", initAllTables = false)
    instance.setModelIfNotSet(spark, tf)
  }

  addReader(readModel)

  def loadSavedModel(modelPath: String, spark: SparkSession): Wav2Vec2ForCTC = {

    val (localModelPath, detectedEngine) = modelSanityCheck(modelPath)

    val vocabJsonContent = loadJsonStringAsset(localModelPath, "vocab.json")
    val vocabJsonMap =
      parse(vocabJsonContent, useBigIntForLong = true).values
        .asInstanceOf[Map[String, BigInt]]

    val preprocessorConfigJsonContent =
      loadJsonStringAsset(localModelPath, "preprocessor_config.json")
    val preprocessorConfig =
      Preprocessor.loadPreprocessorConfig(preprocessorConfigJsonContent)

    /*Universal parameters for all engines*/
    val annotatorModel = new Wav2Vec2ForCTC()
      .setVocabulary(vocabJsonMap)
      .setDoNormalize(preprocessorConfig.do_normalize)
      .setFeatureSize(preprocessorConfig.feature_size)
      .setPaddingSide(preprocessorConfig.padding_side)
      .setPaddingValue(preprocessorConfig.padding_value)
      .setReturnAttentionMask(preprocessorConfig.return_attention_mask)
      .setSamplingRate(preprocessorConfig.sampling_rate)

    annotatorModel.set(annotatorModel.engine, detectedEngine)

    detectedEngine match {
      case ModelEngine.tensorflow =>
        val (wrapper, signatures) =
          TensorflowWrapper.read(localModelPath, zipped = false, useBundle = true)

        val _signatures = signatures match {
          case Some(s) => s
          case None => throw new Exception("Cannot load signature definitions from model!")
        }

        /** the order of setSignatures is important if we use getSignatures inside
          * setModelIfNotSet
          */
        annotatorModel
          .setSignatures(_signatures)
          .setModelIfNotSet(spark, wrapper)

      case _ =>
        throw new Exception(notSupportedEngineError)
    }

    annotatorModel
  }
}

/** This is the companion object of [[Wav2Vec2ForCTC]]. Please refer to that class for the
  * documentation.
  */
object Wav2Vec2ForCTC
    extends ReadablePretrainedWav2Vec2ForAudioModel
    with ReadWav2Vec2ForAudioDLModel

1	/*
2	* Copyright 2017-2022 John Snow Labs
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16
17	package com.johnsnowlabs.nlp.annotators.audio
18
19	import com.johnsnowlabs.ml.ai.Wav2Vec2
20	import com.johnsnowlabs.ml.tensorflow.{
21	ReadTensorflowModel,
22	TensorflowWrapper,
23	WriteTensorflowModel
24	}
25	import com.johnsnowlabs.ml.util.LoadExternalModel.{
26	loadJsonStringAsset,
27	modelSanityCheck,
28	notSupportedEngineError
29	}
30	import com.johnsnowlabs.ml.util.ModelEngine
31	import com.johnsnowlabs.nlp.AnnotatorType.{AUDIO, DOCUMENT}
32	import com.johnsnowlabs.nlp._
33	import com.johnsnowlabs.nlp.annotators.audio.feature_extractor.Preprocessor
34	import com.johnsnowlabs.nlp.serialization.MapFeature
35	import org.apache.spark.broadcast.Broadcast
36	import org.apache.spark.ml.param.IntArrayParam
37	import org.apache.spark.ml.util.Identifiable
38	import org.apache.spark.sql.SparkSession
39	import org.json4s._
40	import org.json4s.jackson.JsonMethods._
41
42	/** Wav2Vec2 Model with a language modeling head on top for Connectionist Temporal Classification
43	* (CTC). Wav2Vec2 was proposed in wav2vec 2.0: A Framework for Self-Supervised Learning of
44	* Speech Representations by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael Auli.
45	*
46	* The annotator takes audio files and transcribes it as text. The audio needs to be provided
47	* pre-processed an array of floats.
48	*
49	* Note that this annotator is currently not supported on Apple Silicon processors such as the
50	* M1/M2 (Apple Silicon). This is due to the processor not supporting instructions for XLA.
51	*
52	* Pretrained models can be loaded with `pretrained` of the companion object:
53	* {{{
54	* val speechToText = Wav2Vec2ForCTC.pretrained()
55	* .setInputCols("audio_assembler")
56	* .setOutputCol("text")
57	* }}}
58	* The default model is `"asr_wav2vec2_base_960h"`, if no name is provided.
59	*
60	* For available pretrained models please see the [[https://sparknlp.org/models Models Hub]].
61	*
62	* To see which models are compatible and how to import them see
63	* [[https://github.com/JohnSnowLabs/spark-nlp/discussions/5669]] and to see more extended
64	* examples, see
65	* [[https://github.com/JohnSnowLabs/spark-nlp/blob/master/src/test/scala/com/johnsnowlabs/nlp/annotators/audio/Wav2Vec2ForCTCTestSpec.scala Wav2Vec2ForCTCTestSpec]].
66	*
67	* ==Example==
68	* {{{
69	* import spark.implicits._
70	* import com.johnsnowlabs.nlp.base._
71	* import com.johnsnowlabs.nlp.annotators._
72	* import com.johnsnowlabs.nlp.annotators.audio.Wav2Vec2ForCTC
73	* import org.apache.spark.ml.Pipeline
74	*
75	* val audioAssembler: AudioAssembler = new AudioAssembler()
76	* .setInputCol("audio_content")
77	* .setOutputCol("audio_assembler")
78	*
79	* val speechToText: Wav2Vec2ForCTC = Wav2Vec2ForCTC
80	* .pretrained()
81	* .setInputCols("audio_assembler")
82	* .setOutputCol("text")
83	*
84	* val pipeline: Pipeline = new Pipeline().setStages(Array(audioAssembler, speechToText))
85	*
86	* val bufferedSource =
87	* scala.io.Source.fromFile("src/test/resources/audio/csv/audio_floats.csv")
88	*
89	* val rawFloats = bufferedSource
90	* .getLines()
91	* .map(_.split(",").head.trim.toFloat)
92	* .toArray
93	* bufferedSource.close
94	*
95	* val processedAudioFloats = Seq(rawFloats).toDF("audio_content")
96	*
97	* val result = pipeline.fit(processedAudioFloats).transform(processedAudioFloats)
98	* result.select("text.result").show(truncate = false)
99	* +------------------------------------------------------------------------------------------+
100	* \|result \|
101	* +------------------------------------------------------------------------------------------+
102	* \|[MISTER QUILTER IS THE APOSTLE OF THE MIDLE CLASES AND WE ARE GLAD TO WELCOME HIS GOSPEL ]\|
103	* +------------------------------------------------------------------------------------------+
104	* }}}
105	* @param uid
106	* required uid for storing annotator to disk
107	* @groupname anno Annotator types
108	* @groupdesc anno
109	* Required input and expected output annotator types
110	* @groupname Ungrouped Members
111	* @groupname param Parameters
112	* @groupname setParam Parameter setters
113	* @groupname getParam Parameter getters
114	* @groupname Ungrouped Members
115	* @groupprio param 1
116	* @groupprio anno 2
117	* @groupprio Ungrouped 3
118	* @groupprio setParam 4
119	* @groupprio getParam 5
120	* @groupdesc param
121	* A list of (hyper-)parameter keys this annotator can take. Users can set and get the
122	* parameter values through setters and getters, respectively.
123	*/
124	class Wav2Vec2ForCTC(override val uid: String)
125	extends AnnotatorModel[Wav2Vec2ForCTC]
126	with HasBatchedAnnotateAudio[Wav2Vec2ForCTC]
127	with HasAudioFeatureProperties
128	with WriteTensorflowModel
129	with HasEngine {
130
131	/** Annotator reference id. Used to identify elements in metadata or to refer to this annotator
132	* type
133	*/
134	def this() = this(Identifiable.randomUID("Wav2Vec2ForCTC"))	×
135
136	/** Output annotator type : DOCUMENT
137	*
138	* @group anno
139	*/
140	override val outputAnnotatorType: AnnotatorType = DOCUMENT	×
141
142	/** Input annotator type : AUDIO
143	*
144	* @group anno
145	*/
146	override val inputAnnotatorTypes: Array[AnnotatorType] = Array(AUDIO)	×
147
148	/** ConfigProto from tensorflow, serialized into byte array. Get with
149	* config_proto.SerializeToString()
150	*
151	* @group param
152	*/
153	val configProtoBytes = new IntArrayParam(	×
154	this,
155	"configProtoBytes",	×
156	"ConfigProto from tensorflow, serialized into byte array. Get with config_proto.SerializeToString()")	×
157
158	/** ConfigProto from tensorflow, serialized into byte array. Get with
159	* config_proto.SerializeToString()
160	*
161	* @group setParam
162	*/
163	def setConfigProtoBytes(bytes: Array[Int]): Wav2Vec2ForCTC.this.type =
164	set(this.configProtoBytes, bytes)	×
165
166	/** ConfigProto from tensorflow, serialized into byte array. Get with
167	* config_proto.SerializeToString()
168	*
169	* @group getParam
170	*/
171	def getConfigProtoBytes: Option[Array[Byte]] =
172	get(this.configProtoBytes).map(_.map(_.toByte))	×
173
174	/** Vocabulary used to encode the words to ids
175	*
176	* @group param
177	*/
178	val vocabulary: MapFeature[String, BigInt] = new MapFeature(this, "vocabulary").setProtected()	×
179
180	/** @group setParam */
181	def setVocabulary(value: Map[String, BigInt]): this.type = set(vocabulary, value)	×
182
183	/** It contains TF model signatures for the laded saved model
184	*
185	* @group param
186	*/
187	val signatures =
188	new MapFeature[String, String](model = this, name = "signatures").setProtected()	×
189
190	/** @group setParam */
191	def setSignatures(value: Map[String, String]): this.type = {
192	set(signatures, value)	×
193	this
194	}
195
196	/** @group getParam */
197	def getSignatures: Option[Map[String, String]] = get(this.signatures)	×
198
199	private var _model: Option[Broadcast[Wav2Vec2]] = None	×
200
201	/** @group getParam */
202	def getModelIfNotSet: Wav2Vec2 = _model.get.value	×
203
204	/** @group setParam */
205	def setModelIfNotSet(spark: SparkSession, tensorflow: TensorflowWrapper): this.type = {
206	if (_model.isEmpty) {	×
207
208	_model = Some(	×
209	spark.sparkContext.broadcast(	×
210	new Wav2Vec2(	×
211	tensorflow,
212	configProtoBytes = getConfigProtoBytes,	×
213	vocabs = $$(vocabulary),	×
214	signatures = getSignatures)))	×
215	}
216	this
217	}
218
219	setDefault(batchSize -> 4)	×
220
221	/** Takes a document and annotations and produces new annotations of this annotator's annotation
222	* type
223	*
224	* @param batchedAnnotations
225	* Annotations that correspond to inputAnnotationCols generated by previous annotators if any
226	* @return
227	* any number of annotations processed for every input annotation. Not necessary one to one
228	* relationship
229	*/
230	override def batchAnnotate(
231	batchedAnnotations: Seq[Array[AnnotationAudio]]): Seq[Seq[Annotation]] = {
232
233	// Zip annotations to the row it belongs to
234	val audiosWithRow = batchedAnnotations.zipWithIndex	×
235	.flatMap { case (annotations, i) => annotations.map(x => (x, i)) }	×
236
237	val noneEmptyAudios = audiosWithRow.map(_._1).filter(_.result.nonEmpty).toArray	×
238
239	val allAnnotations =
240	if (noneEmptyAudios.nonEmpty) {	×
241	getModelIfNotSet.predict(	×
242	audios = noneEmptyAudios,
243	batchSize = $(batchSize),	×
244	preprocessor = Preprocessor(	×
245	do_normalize = getDoNormalize,	×
246	return_attention_mask = getReturnAttentionMask,	×
247	padding_side = getPaddingSide,	×
248	padding_value = getPaddingValue,	×
249	feature_size = getFeatureSize,	×
250	sampling_rate = getSamplingRate))	×
251	} else {
252	Seq.empty[Annotation]	×
253	}
254
255	// Group resulting annotations by rows. If there are not sentences in a given row, return empty sequence
256	batchedAnnotations.indices.map(rowIndex => {	×
257	val rowAnnotations = allAnnotations
258	// zip each annotation with its corresponding row index
259	.zip(audiosWithRow)	×
260	// select the sentences belonging to the current row
261	.filter(_._2._2 == rowIndex)	×
262	// leave the annotation only
263	.map(_._1)	×
264
265	if (rowAnnotations.nonEmpty)	×
266	rowAnnotations	×
267	else
268	Seq.empty[Annotation]	×
269	})
270
271	}
272
273	override def onWrite(path: String, spark: SparkSession): Unit = {
274	super.onWrite(path, spark)	×
275	writeTensorflowModelV2(	×
276	path,
277	spark,
278	getModelIfNotSet.tensorflowWrapper,	×
279	"_wav_ctc",	×
280	Wav2Vec2ForCTC.tfFile,	×
281	configProtoBytes = getConfigProtoBytes)	×
282	}
283
284	}
285
286	trait ReadablePretrainedWav2Vec2ForAudioModel
287	extends ParamsAndFeaturesReadable[Wav2Vec2ForCTC]
288	with HasPretrained[Wav2Vec2ForCTC] {
289	override val defaultModelName: Some[String] = Some("asr_wav2vec2_base_960h")	×
290
291	/** Java compliant-overrides */
292	override def pretrained(): Wav2Vec2ForCTC = super.pretrained()	×
293
294	override def pretrained(name: String): Wav2Vec2ForCTC = super.pretrained(name)	×
295
296	override def pretrained(name: String, lang: String): Wav2Vec2ForCTC =
297	super.pretrained(name, lang)	×
298
299	override def pretrained(name: String, lang: String, remoteLoc: String): Wav2Vec2ForCTC =
300	super.pretrained(name, lang, remoteLoc)	×
301	}
302
303	trait ReadWav2Vec2ForAudioDLModel extends ReadTensorflowModel {
304	this: ParamsAndFeaturesReadable[Wav2Vec2ForCTC] =>
305
306	override val tfFile: String = "wav_ctc_tensorflow"	×
307
308	def readModel(instance: Wav2Vec2ForCTC, path: String, spark: SparkSession): Unit = {
309
310	val tf = readTensorflowModel(path, spark, "_wav_ctc_tf", initAllTables = false)	×
311	instance.setModelIfNotSet(spark, tf)	×
312	}
313
314	addReader(readModel)	×
315
316	def loadSavedModel(modelPath: String, spark: SparkSession): Wav2Vec2ForCTC = {
317
318	val (localModelPath, detectedEngine) = modelSanityCheck(modelPath)	×
319
320	val vocabJsonContent = loadJsonStringAsset(localModelPath, "vocab.json")	×
321	val vocabJsonMap =
322	parse(vocabJsonContent, useBigIntForLong = true).values	×
323	.asInstanceOf[Map[String, BigInt]]	×
324
325	val preprocessorConfigJsonContent =
326	loadJsonStringAsset(localModelPath, "preprocessor_config.json")	×
327	val preprocessorConfig =
328	Preprocessor.loadPreprocessorConfig(preprocessorConfigJsonContent)	×
329
330	/Universal parameters for all engines/
331	val annotatorModel = new Wav2Vec2ForCTC()
332	.setVocabulary(vocabJsonMap)
333	.setDoNormalize(preprocessorConfig.do_normalize)	×
334	.setFeatureSize(preprocessorConfig.feature_size)	×
335	.setPaddingSide(preprocessorConfig.padding_side)	×
336	.setPaddingValue(preprocessorConfig.padding_value)	×
337	.setReturnAttentionMask(preprocessorConfig.return_attention_mask)	×
338	.setSamplingRate(preprocessorConfig.sampling_rate)	×
339
340	annotatorModel.set(annotatorModel.engine, detectedEngine)	×
341
342	detectedEngine match {
343	case ModelEngine.tensorflow =>
344	val (wrapper, signatures) =	×
345	TensorflowWrapper.read(localModelPath, zipped = false, useBundle = true)
346
347	val _signatures = signatures match {
348	case Some(s) => s
349	case None => throw new Exception("Cannot load signature definitions from model!")	×
350	}
351
352	/** the order of setSignatures is important if we use getSignatures inside
353	* setModelIfNotSet
354	*/
355	annotatorModel
356	.setSignatures(_signatures)
357	.setModelIfNotSet(spark, wrapper)	×
358
359	case _ =>
360	throw new Exception(notSupportedEngineError)	×
361	}
362
363	annotatorModel
364	}
365	}
366
367	/** This is the companion object of [[Wav2Vec2ForCTC]]. Please refer to that class for the
368	* documentation.
369	*/
370	object Wav2Vec2ForCTC
371	extends ReadablePretrainedWav2Vec2ForAudioModel
372	with ReadWav2Vec2ForAudioDLModel

JohnSnowLabs / spark-nlp / 4992350528

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous