10656334014

Committed 01 Sep 2024 06:19PM CUT coverage: 62.392% (-0.02%) from 62.41%

Build # 10656334014

Build Type

Pull #14355

github

Committed by

web-flow

Commit Message

Merge 2a3ee298b into 50a69662f

Pull Request Pull Request #14355: Implementing Mxbai Embeddings

Run Details

0 of 2 new or added lines in 1 file covered. (0.0%)

27 existing lines in 7 files now uncovered.

8967 of 14372 relevant lines covered (62.39%)

0.62 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.33

/src/main/scala/com/johnsnowlabs/util/ZipArchiveUtil.scala

/*
 * Copyright 2017-2022 John Snow Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.johnsnowlabs.util

import org.apache.commons.io.FileUtils

import java.io._
import java.util.zip.{ZipEntry, ZipFile, ZipOutputStream}
import scala.collection.JavaConverters._

object ZipArchiveUtil {
  // Recursively lists all files in a given directory, returning a list of their absolute paths
  private[util] def listFilesRecursive(file: File): List[File] = {
    file match {
      case file if file.isFile => List(new File(file.getAbsoluteFile.toString))
      case file if file.isDirectory =>
        val fList = file.list
        // Add all files in current dir to list and recur on subdirs
        fList.foldLeft(List[File]())((pList: List[File], path: String) =>
          pList ++ listFilesRecursive(new File(file, path)))
      case _ => throw new IOException("Bad path. No file or directory found.")
    }
  }

  private[util] def addFileToZipEntry(
      filename: File,
      parentPath: File,
      useRelativePath: Boolean = false): ZipEntry = {
    if (!useRelativePath) // use absolute path
      new ZipEntry(filename.getName)
    else { // use relative path
      val relative = parentPath.toURI.relativize(filename.toURI).getPath
      new ZipEntry(relative)
    }
  }

  private[util] def createZip(
      filePaths: List[File],
      outputFilePath: File,
      parentPath: File): Unit = {

    val Buffer = 2 * 1024
    val data = new Array[Byte](Buffer)
    try {
      val zipFileOS = new FileOutputStream(outputFilePath)
      val zip = new ZipOutputStream(zipFileOS)
      zip.setLevel(0)
      filePaths.foreach((file: File) => {
        val zipEntry = addFileToZipEntry(file, parentPath, filePaths.size > 1)
        // add zip entry to output stream
        zip.putNextEntry(new ZipEntry(zipEntry))
        val in = new BufferedInputStream(new FileInputStream(file), Buffer)
        var b = in.read(data, 0, Buffer)
        while (b != -1) {
          zip.write(data, 0, b)
          b = in.read(data, 0, Buffer)
        }
        in.close()
      })
      zip.closeEntry()
      zip.close()
      zipFileOS.close()

    } catch {
      case e: IOException =>
        e.printStackTrace()
    }
  }

  private[util] def zipFile(soureFile: File, outputFilePath: File): Unit = {
    createZip(List(soureFile.getAbsoluteFile), outputFilePath, null)
  }

  private[util] def zipDir(sourceDir: File, outputFilePath: File): Unit = {
    val filePaths = listFilesRecursive(sourceDir)
    createZip(filePaths, outputFilePath, sourceDir)
  }

  def zip(sourcePath: String, outputFilePath: String): Unit = {
    val sourceFile = new File(sourcePath)
    val outputFile = new File(outputFilePath)
    if (sourceFile.equals(outputFile))
      throw new IllegalArgumentException("source path cannot be identical to target path")

    if (!outputFile.getParentFile().exists)
      throw new IOException("the parent directory of output file doesn't exist")

    if (!sourceFile.exists())
      throw new IOException("zip source path must exsit")

    if (outputFile.exists())
      throw new IOException("zip target file exsits")

    if (sourceFile.isDirectory())
      zipDir(sourceFile, outputFile)
    else if (sourceFile.isFile())
      zipFile(sourceFile, outputFile)
    else
      throw new IllegalArgumentException("only folder and file input are valid")
  }

  def unzip(
      file: File,
      destDirPath: Option[String] = None,
      suffix: Option[String] = None): String = {
    val fileName = file.getName

    val basename = if (fileName.indexOf('.') >= 0) {
      fileName.substring(0, fileName.lastIndexOf("."))
    } else {
      fileName + "_unzipped"
    }

    val destDir = if (destDirPath.isEmpty) {
      new File(file.getParentFile, basename)
    } else {
      new File(destDirPath.get)
    }

    destDir.mkdirs()

    val zip = new ZipFile(file)
    zip.entries.asScala foreach { entry =>
      val entryName = buildEntryName(entry, suffix)
      val entryPath = {
        if (entryName.startsWith(basename))
          entryName.substring(0, basename.length)
        else
          entryName
      }

      // create output directory if it doesn't exist already
      val toDrop = if (entry.isDirectory) 0 else 1
      val splitPath = entryName.split(File.separator.replace("\\", "/")).dropRight(toDrop)

      val dirBuilder = new StringBuilder(destDir.getPath)
      for (part <- splitPath) {
        dirBuilder.append(File.separator)
        dirBuilder.append(part)
        val path = dirBuilder.toString

        if (!new File(path).exists) {
          new File(path).mkdir
        }
      }

      // write file to dest
      FileUtils.copyInputStreamToFile(zip.getInputStream(entry), new File(destDir, entryPath))
    }

    destDir.getPath
  }

  private def buildEntryName(entry: ZipEntry, suffix: Option[String]): String = {
    val entryName = if (suffix.isDefined) suffix.get + "_" + entry.getName else entry.getName
    entryName.split("_").distinct.mkString("_")
  }

}

1	/*
2	* Copyright 2017-2022 John Snow Labs
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16
17	package com.johnsnowlabs.util
18
19	import org.apache.commons.io.FileUtils
20
21	import java.io._
22	import java.util.zip.{ZipEntry, ZipFile, ZipOutputStream}
23	import scala.collection.JavaConverters._
24
25	object ZipArchiveUtil {
26	// Recursively lists all files in a given directory, returning a list of their absolute paths
27	private[util] def listFilesRecursive(file: File): List[File] = {
28	file match {
29	case file if file.isFile => List(new File(file.getAbsoluteFile.toString))	1✔
30	case file if file.isDirectory =>	1✔
31	val fList = file.list	1✔
32	// Add all files in current dir to list and recur on subdirs
33	fList.foldLeft(List[File]())((pList: List[File], path: String) =>	1✔
34	pList ++ listFilesRecursive(new File(file, path)))	1✔
35	case _ => throw new IOException("Bad path. No file or directory found.")	1✔
36	}
37	}
38
39	private[util] def addFileToZipEntry(
40	filename: File,
41	parentPath: File,
42	useRelativePath: Boolean = false): ZipEntry = {
43	if (!useRelativePath) // use absolute path	1✔
44	new ZipEntry(filename.getName)	1✔
45	else { // use relative path	1✔
46	val relative = parentPath.toURI.relativize(filename.toURI).getPath	1✔
47	new ZipEntry(relative)	1✔
48	}
49	}
50
51	private[util] def createZip(
52	filePaths: List[File],
53	outputFilePath: File,
54	parentPath: File): Unit = {
55
56	val Buffer = 2 * 1024	1✔
57	val data = new Array[Byte](Buffer)	1✔
58	try {	1✔
59	val zipFileOS = new FileOutputStream(outputFilePath)	1✔
60	val zip = new ZipOutputStream(zipFileOS)	1✔
61	zip.setLevel(0)	1✔
62	filePaths.foreach((file: File) => {	1✔
63	val zipEntry = addFileToZipEntry(file, parentPath, filePaths.size > 1)	1✔
64	// add zip entry to output stream
65	zip.putNextEntry(new ZipEntry(zipEntry))	1✔
66	val in = new BufferedInputStream(new FileInputStream(file), Buffer)	1✔
67	var b = in.read(data, 0, Buffer)	1✔
68	while (b != -1) {	1✔
69	zip.write(data, 0, b)	1✔
70	b = in.read(data, 0, Buffer)	1✔
71	}
72	in.close()	1✔
73	})
74	zip.closeEntry()	1✔
75	zip.close()	1✔
76	zipFileOS.close()	1✔
77
78	} catch {
79	case e: IOException =>
80	e.printStackTrace()	×
81	}
82	}
83
84	private[util] def zipFile(soureFile: File, outputFilePath: File): Unit = {
85	createZip(List(soureFile.getAbsoluteFile), outputFilePath, null)	1✔
86	}
87
88	private[util] def zipDir(sourceDir: File, outputFilePath: File): Unit = {
89	val filePaths = listFilesRecursive(sourceDir)	1✔
90	createZip(filePaths, outputFilePath, sourceDir)	1✔
91	}
92
93	def zip(sourcePath: String, outputFilePath: String): Unit = {
94	val sourceFile = new File(sourcePath)	1✔
95	val outputFile = new File(outputFilePath)	1✔
96	if (sourceFile.equals(outputFile))	1✔
97	throw new IllegalArgumentException("source path cannot be identical to target path")	×
98
99	if (!outputFile.getParentFile().exists)	1✔
100	throw new IOException("the parent directory of output file doesn't exist")	1✔
101
102	if (!sourceFile.exists())	1✔
103	throw new IOException("zip source path must exsit")	×
104
105	if (outputFile.exists())	1✔
106	throw new IOException("zip target file exsits")	×
107
108	if (sourceFile.isDirectory())	1✔
109	zipDir(sourceFile, outputFile)	1✔
110	else if (sourceFile.isFile())	1✔
111	zipFile(sourceFile, outputFile)	1✔
112	else
113	throw new IllegalArgumentException("only folder and file input are valid")	×
114	}
115
116	def unzip(
117	file: File,
118	destDirPath: Option[String] = None,
119	suffix: Option[String] = None): String = {
120	val fileName = file.getName	1✔
121
122	val basename = if (fileName.indexOf('.') >= 0) {	1✔
123	fileName.substring(0, fileName.lastIndexOf("."))	×
124	} else {
125	fileName + "_unzipped"	1✔
126	}
127
128	val destDir = if (destDirPath.isEmpty) {	1✔
129	new File(file.getParentFile, basename)	×
130	} else {
131	new File(destDirPath.get)	1✔
132	}
133
134	destDir.mkdirs()	1✔
135
136	val zip = new ZipFile(file)	1✔
137	zip.entries.asScala foreach { entry =>	1✔
138	val entryName = buildEntryName(entry, suffix)	1✔
139	val entryPath = {
140	if (entryName.startsWith(basename))	1✔
141	entryName.substring(0, basename.length)	×
142	else
143	entryName	1✔
144	}
145
146	// create output directory if it doesn't exist already
UNCOV 147	val toDrop = if (entry.isDirectory) 0 else 1	×
148	val splitPath = entryName.split(File.separator.replace("\\", "/")).dropRight(toDrop)	1✔
149
150	val dirBuilder = new StringBuilder(destDir.getPath)	1✔
151	for (part <- splitPath) {	1✔
152	dirBuilder.append(File.separator)	1✔
153	dirBuilder.append(part)	1✔
154	val path = dirBuilder.toString	1✔
155
156	if (!new File(path).exists) {	×
157	new File(path).mkdir	1✔
158	}
159	}
160
161	// write file to dest
162	FileUtils.copyInputStreamToFile(zip.getInputStream(entry), new File(destDir, entryPath))	1✔
163	}
164
165	destDir.getPath	1✔
166	}
167
168	private def buildEntryName(entry: ZipEntry, suffix: Option[String]): String = {
UNCOV 169	val entryName = if (suffix.isDefined) suffix.get + "_" + entry.getName else entry.getName	×
170	entryName.split("_").distinct.mkString("_")	1✔
171	}
172
173	}

JohnSnowLabs / spark-nlp / 10656334014

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous