19799834564

Committed 30 Nov 2025 01:47PM UTC coverage: 68.849% (+0.002%) from 68.847%

Build # 19799834564

Build Type

push

github

Committed by

bpatrik

Commit Message

Improving saving loop handling #1080

Run Details

1452 of 2363 branches covered (61.45%)

Branch coverage included in aggregate %.

8 of 9 new or added lines in 1 file covered. (88.89%)

1 existing line in 1 file now uncovered.

5278 of 7412 relevant lines covered (71.21%)

4164.6 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.53

/src/backend/model/database/IndexingManager.ts

import {DirectoryBaseDTO, DirectoryDTOUtils, DirectoryPathDTO, ParentDirectoryDTO} from '../../../common/entities/DirectoryDTO';
import {DirectoryEntity} from './enitites/DirectoryEntity';
import {SQLConnection} from './SQLConnection';
import {PhotoEntity, PhotoMetadataEntity} from './enitites/PhotoEntity';
import {Utils} from '../../../common/Utils';
import {PhotoMetadata,} from '../../../common/entities/PhotoDTO';
import {Connection, ObjectLiteral, Repository} from 'typeorm';
import {MediaEntity} from './enitites/MediaEntity';
import {MediaDTO, MediaDTOUtils} from '../../../common/entities/MediaDTO';
import {VideoEntity} from './enitites/VideoEntity';
import {FileEntity} from './enitites/FileEntity';
import {FileDTO} from '../../../common/entities/FileDTO';
import {NotificationManager} from '../NotifocationManager';
import {ObjectManagers} from '../ObjectManagers';
import {Logger} from '../../Logger';
import {ServerPG2ConfMap, ServerSidePG2ConfAction,} from '../../../common/PG2ConfMap';
import {ProjectPath} from '../../ProjectPath';
import * as path from 'path';
import * as fs from 'fs';
import {SearchQueryDTO} from '../../../common/entities/SearchQueryDTO';
import {PersonEntry} from './enitites/person/PersonEntry';
import {PersonJunctionTable} from './enitites/person/PersonJunctionTable';
import {MDFileEntity} from './enitites/MDFileEntity';
import {MDFileDTO} from '../../../common/entities/MDFileDTO';
import {DiskManager} from '../fileaccess/DiskManager';
import {ProjectedDirectoryCacheEntity} from './enitites/ProjectedDirectoryCacheEntity';

const LOG_TAG = '[IndexingManager]';

export class IndexingManager {
  SavingReady: Promise<void> = null;
  private SavingReadyPR: () => void = null;
  private savingQueue: { dir: ParentDirectoryDTO; promise: Promise<void>; resolve: () => void; reject: (e: any) => void }[] = [];
  private isSaving = false;

  get IsSavingInProgress(): boolean {
    return this.SavingReady !== null;
  }

  private static async processServerSidePG2Conf(
    parent: DirectoryPathDTO,
    files: FileDTO[]
  ): Promise<void> {
    for (const f of files) {
      if (ServerPG2ConfMap[f.name] === ServerSidePG2ConfAction.SAVED_SEARCH) {
        const fullMediaPath = path.join(
          ProjectPath.ImageFolder,
          parent.path,
          parent.name,
          f.name
        );

        Logger.silly(
          LOG_TAG,
          'Saving saved-searches to DB from:',
          fullMediaPath
        );
        const savedSearches: { name: string; searchQuery: SearchQueryDTO }[] =
          JSON.parse(await fs.promises.readFile(fullMediaPath, 'utf8'));
        for (const s of savedSearches) {
          await ObjectManagers.getInstance().AlbumManager.addIfNotExistSavedSearch(
            s.name,
            s.searchQuery,
            true
          );
        }
      }
    }
  }

  /**
   * Indexes a dir, but returns early with the scanned version,
   * does not wait for the DB to be saved
   */
  public async indexDirectory(
    relativeDirectoryName: string,
    waitForSave = false
  ): Promise<ParentDirectoryDTO> {
    try {
      // Check if root is still a valid (non-empty) folder
      // With weak devices, it is possible that the media that stores
      // the galley gets unmounted that triggers a full gallery wipe.
      // Prevent it by stopping indexing on an empty folder.
      if (fs.readdirSync(ProjectPath.ImageFolder).length === 0) {
        throw new Error('Root directory is empty. This is probably error and would erase gallery database. Stopping indexing.');
      }

      const scannedDirectory = await DiskManager.scanDirectory(
        relativeDirectoryName
      );

      const dirClone = Utils.clone(scannedDirectory);
      // filter server side only config from returning
      dirClone.metaFile = dirClone.metaFile.filter(
        (m) => !ServerPG2ConfMap[m.name]
      );

      DirectoryDTOUtils.addReferences(dirClone);

      if (waitForSave === true) {
        // save directory to DB and wait until saving finishes
        await this.queueForSave(scannedDirectory);
        return dirClone;
      }

      // save directory to DB in the background
      this.queueForSave(scannedDirectory).catch(console.error);
      return dirClone;
    } catch (error) {
      NotificationManager.warning(
        'Unknown indexing error for: ' + relativeDirectoryName,
        error.toString()
      );
      console.error(error);
      throw error;
    }
  }

  async resetDB(): Promise<void> {
    Logger.info(LOG_TAG, 'Resetting DB');
    const connection = await SQLConnection.getConnection();
    await connection
      .getRepository(DirectoryEntity)
      .createQueryBuilder('directory')
      .delete()
      .execute();
  }

  public async saveToDB(scannedDirectory: ParentDirectoryDTO): Promise<void> {
    this.isSaving = true;
    try {
      const connection = await SQLConnection.getConnection();
      const serverSideConfigs = scannedDirectory.metaFile.filter(
        (m) => !!ServerPG2ConfMap[m.name]
      );
      scannedDirectory.metaFile = scannedDirectory.metaFile.filter(
        (m) => !ServerPG2ConfMap[m.name]
      );
      const currentDirId: number = await this.saveParentDir(
        connection,
        scannedDirectory
      );
      await this.saveChildDirs(connection, currentDirId, scannedDirectory);
      await this.saveMedia(connection, currentDirId, scannedDirectory.media);
      await this.saveMetaFiles(connection, currentDirId, scannedDirectory);
      await IndexingManager.processServerSidePG2Conf(scannedDirectory, serverSideConfigs);
      await ObjectManagers.getInstance().onDataChange(scannedDirectory);
    } finally {
      this.isSaving = false;
    }
  }

  // Todo fix it, once typeorm support connection pools for sqlite
  /**
   * Queues up a directory to save to the DB.
   * Returns a promise that resolves when the directory is saved.
   */
  protected async queueForSave(
    scannedDirectory: ParentDirectoryDTO
  ): Promise<void> {
    // Is this dir already queued for saving?
    const existingIndex = this.savingQueue.findIndex(
      (entry): boolean =>
        entry.dir.name === scannedDirectory.name &&
        entry.dir.path === scannedDirectory.path &&
        entry.dir.lastModified === scannedDirectory.lastModified &&
        entry.dir.lastScanned === scannedDirectory.lastScanned &&
        (entry.dir.media || entry.dir.media.length) ===
        (scannedDirectory.media || scannedDirectory.media.length) &&
        (entry.dir.metaFile || entry.dir.metaFile.length) ===
        (scannedDirectory.metaFile || scannedDirectory.metaFile.length)
    );
    if (existingIndex !== -1) {
      return this.savingQueue[existingIndex].promise;
    }

    // queue for saving
    let resolveFn: () => void;
    let rejectFn: (e: any) => void;
    const promise = new Promise<void>((resolve, reject): void => {
      resolveFn = resolve;
      rejectFn = reject;
    });

    this.savingQueue.push({dir: scannedDirectory, promise, resolve: resolveFn, reject: rejectFn});

    if (this.savingQueue.length > 100) {
      Logger.warn(LOG_TAG, 'Saving queue is growing large:', this.savingQueue.length);
    }

    this.runSavingLoop().catch(console.error);

    return promise;
  }

  protected async runSavingLoop(): Promise<void> {

    // start saving if not already started
    if (!this.SavingReady) {
      this.SavingReady = new Promise<void>((resolve): void => {
        this.SavingReadyPR = resolve;
      });
    }
    try {
      while (this.isSaving === false && this.savingQueue.length > 0) {
        const item = this.savingQueue[0];
        try {
          await this.saveToDB(item.dir);
          item.resolve();
        } catch (e) {
          // reject current and remaining queued items to avoid hanging promises
          item.reject(e);
          this.savingQueue.shift();
          for (const remaining of this.savingQueue) {
            remaining.reject(e);
          }
          this.savingQueue = [];
          this.checkSavingReady();
          throw e;
        }
        this.savingQueue.shift();
      }
    } finally {
      this.checkSavingReady();
    }
  }

  protected async saveParentDir(
    connection: Connection,
    scannedDirectory: ParentDirectoryDTO
  ): Promise<number> {
    const directoryRepository = connection.getRepository(DirectoryEntity);
    const projDirCacheRep = connection.getRepository(ProjectedDirectoryCacheEntity);

    const currentDir: DirectoryEntity = await directoryRepository
      .createQueryBuilder('directory')
      .where('directory.name = :name AND directory.path = :path', {
        name: scannedDirectory.name,
        path: scannedDirectory.path,
      })
      .getOne();
    if (currentDir) {
      // Updated parent dir (if it was in the DB previously)
      currentDir.lastModified = scannedDirectory.lastModified;
      currentDir.lastScanned = scannedDirectory.lastScanned;
      await directoryRepository.save(currentDir);
      return currentDir.id;
    } else {
      return (
        await directoryRepository.insert({
          lastModified: scannedDirectory.lastModified,
          lastScanned: scannedDirectory.lastScanned,
          name: scannedDirectory.name,
          path: scannedDirectory.path,
        } as DirectoryEntity)
      ).identifiers[0]['id'];
    }
  }

  protected async saveChildDirs(
    connection: Connection,
    currentDirId: number,
    scannedDirectory: ParentDirectoryDTO
  ): Promise<void> {
    const directoryRepository = connection.getRepository(DirectoryEntity);

    // update subdirectories that does not have a parent
    await directoryRepository
      .createQueryBuilder()
      .update(DirectoryEntity)
      .set({parent: currentDirId as unknown})
      .where('path = :path', {
        path: DiskManager.pathFromParent(scannedDirectory),
      })
      .andWhere('name NOT LIKE :root', {root: DiskManager.dirName('.')})
      .andWhere('parent IS NULL')
      .execute();

    // save subdirectories
    const childDirectories = await directoryRepository
      .createQueryBuilder('directory')
      .leftJoinAndSelect('directory.parent', 'parent')
      .where('directory.parent = :dir', {
        dir: currentDirId,
      })
      .getMany();

    for (const directory of scannedDirectory.directories) {
      // Was this child Dir already indexed before?
      const dirIndex = childDirectories.findIndex(
        (d): boolean => d.name === directory.name
      );

      if (dirIndex !== -1) {
        // directory found
        childDirectories.splice(dirIndex, 1);
      } else {
        // dir does not exist yet
        directory.parent = {id: currentDirId} as ParentDirectoryDTO;
        (directory as DirectoryEntity).lastScanned = null; // new child dir, not fully scanned yet
        const d = await directoryRepository.insert(
          directory as DirectoryEntity
        );

        await this.saveMedia(
          connection,
          d.identifiers[0]['id'],
          directory.media
        );
      }
    }

    // Remove child Dirs that are not anymore in the parent dir
    await directoryRepository.remove(childDirectories, {
      chunk: Math.max(Math.ceil(childDirectories.length / 500), 1),
    });
  }

  protected async saveMetaFiles(
    connection: Connection,
    currentDirID: number,
    scannedDirectory: ParentDirectoryDTO
  ): Promise<void> {
    const fileRepository = connection.getRepository(FileEntity);
    const MDfileRepository = connection.getRepository(MDFileEntity);
    // save files
    const indexedMetaFiles = await fileRepository
      .createQueryBuilder('file')
      .where('file.directory = :dir', {
        dir: currentDirID,
      })
      .getMany();

    const metaFilesToInsert = [];
    const MDFilesToUpdate = [];
    for (const item of scannedDirectory.metaFile) {
      let metaFile: FileDTO = null;
      for (let j = 0; j < indexedMetaFiles.length; j++) {
        if (indexedMetaFiles[j].name === item.name) {
          metaFile = indexedMetaFiles[j];
          indexedMetaFiles.splice(j, 1);
          break;
        }
      }
      if (metaFile == null) {
        // not in DB yet
        item.directory = null;
        metaFile = Utils.clone(item);
        item.directory = scannedDirectory;
        metaFile.directory = {id: currentDirID} as DirectoryBaseDTO;
        metaFilesToInsert.push(metaFile);
      } else if ((item as MDFileDTO).date) {
        if ((item as MDFileDTO).date != (metaFile as MDFileDTO).date) {
          (metaFile as MDFileDTO).date = (item as MDFileDTO).date;
          MDFilesToUpdate.push(metaFile);
        }
      }
    }

    const MDFiles = metaFilesToInsert.filter(f => !isNaN((f as MDFileDTO).date));
    const generalFiles = metaFilesToInsert.filter(f => isNaN((f as MDFileDTO).date));
    await fileRepository.save(generalFiles, {
      chunk: Math.max(Math.ceil(generalFiles.length / 500), 1),
    });
    await MDfileRepository.save(MDFiles, {
      chunk: Math.max(Math.ceil(MDFiles.length / 500), 1),
    });
    await MDfileRepository.save(MDFilesToUpdate, {
      chunk: Math.max(Math.ceil(MDFilesToUpdate.length / 500), 1),
    });
    await fileRepository.remove(indexedMetaFiles, {
      chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1),
    });
  }

  protected async saveMedia(
    connection: Connection,
    parentDirId: number,
    media: MediaDTO[]
  ): Promise<void> {
    const mediaRepository = connection.getRepository(MediaEntity);
    const photoRepository = connection.getRepository(PhotoEntity);
    const videoRepository = connection.getRepository(VideoEntity);
    // save media
    let indexedMedia = await mediaRepository
      .createQueryBuilder('media')
      .where('media.directory = :dir', {
        dir: parentDirId,
      })
      .getMany();

    const mediaChange = {
      saveP: [] as MediaDTO[], // save/update photo
      saveV: [] as MediaDTO[], // save/update video
      insertP: [] as MediaDTO[], // insert photo
      insertV: [] as MediaDTO[], // insert video
    };
    const personsPerPhoto: { faces: { name: string, mediaId?: number }[]; mediaName: string }[] = [];
    // eslint-disable-next-line @typescript-eslint/prefer-for-of
    for (let i = 0; i < media.length; i++) {
      let mediaItem: MediaDTO = null;
      for (let j = 0; j < indexedMedia.length; j++) {
        if (indexedMedia[j].name === media[i].name) {
          mediaItem = indexedMedia[j];
          indexedMedia.splice(j, 1);
          break;
        }
      }

      const scannedFaces: { name: string }[] = (media[i].metadata as PhotoMetadata).faces || [];
      if ((media[i].metadata as PhotoMetadata).faces) {
        // if it has faces, cache them
        // make the list distinct (some photos may contain the same person multiple times)
        (media[i].metadata as PhotoMetadataEntity).persons = [
          ...new Set(
            (media[i].metadata as PhotoMetadata).faces.map((f) => f.name)
          ),
        ];
      }
      (media[i].metadata as PhotoMetadataEntity).personsLength = (media[i].metadata as PhotoMetadataEntity)?.persons?.length || 0;


      if (mediaItem == null) {
        // Media not in DB yet
        media[i].directory = null;
        mediaItem = Utils.clone(media[i]);
        mediaItem.directory = {id: parentDirId} as DirectoryBaseDTO;
        (MediaDTOUtils.isPhoto(mediaItem)
            ? mediaChange.insertP
            : mediaChange.insertV
        ).push(mediaItem);
      } else {
        // Media already in the DB, only needs to be updated
        delete (mediaItem.metadata as PhotoMetadata).faces;
        if (!Utils.equalsFilter(mediaItem.metadata, media[i].metadata)) {
          mediaItem.metadata = media[i].metadata;
          (MediaDTOUtils.isPhoto(mediaItem)
              ? mediaChange.saveP
              : mediaChange.saveV
          ).push(mediaItem);
        }
      }

      personsPerPhoto.push({
        faces: scannedFaces,
        mediaName: mediaItem.name
      });
    }

    await this.saveChunk(photoRepository, mediaChange.saveP, 100);
    await this.saveChunk(videoRepository, mediaChange.saveV, 100);
    await this.saveChunk(photoRepository, mediaChange.insertP, 100);
    await this.saveChunk(videoRepository, mediaChange.insertV, 100);

    indexedMedia = await mediaRepository
      .createQueryBuilder('media')
      .where('media.directory = :dir', {
        dir: parentDirId,
      })
      .select(['media.name', 'media.id'])
      .getMany();

    const persons: { name: string; mediaId: number }[] = [];
    personsPerPhoto.forEach((group): void => {
      const mIndex = indexedMedia.findIndex(
        (m): boolean => m.name === group.mediaName
      );
      group.faces.forEach((sf) =>
        (sf.mediaId = indexedMedia[mIndex].id)
      );

      persons.push(...group.faces as { name: string; mediaId: number }[]);
      indexedMedia.splice(mIndex, 1);
    });

    await this.savePersonsToMedia(connection, parentDirId, persons);
    await mediaRepository.remove(indexedMedia);
  }

  protected async savePersonsToMedia(
    connection: Connection,
    parentDirId: number,
    scannedFaces: { name: string; mediaId: number }[]
  ): Promise<void> {
    const personJunctionTable = connection.getRepository(PersonJunctionTable);
    const personRepository = connection.getRepository(PersonEntry);

    const persons: { name: string; mediaId: number }[] = [];

    // Make a set
    for (const face of scannedFaces) {
      if (persons.findIndex((f) => f.name === face.name) === -1) {
        persons.push(face);
      }
    }
    await ObjectManagers.getInstance().PersonManager.saveAll(persons);
    // get saved persons without triggering denormalized data update (i.e.: do not use PersonManager.get).
    const savedPersons = await personRepository.find();

    const indexedFaces = await personJunctionTable
      .createQueryBuilder('face')
      .leftJoin('face.media', 'media')
      .where('media.directory = :directory', {
        directory: parentDirId,
      })
      .leftJoinAndSelect('face.person', 'person')
      .getMany();

    const faceToInsert: { person: { id: number }, media: { id: number } }[] = [];
    // eslint-disable-next-line @typescript-eslint/prefer-for-of
    for (let i = 0; i < scannedFaces.length; i++) {
      // was the Person - media connection already indexed
      let face: PersonJunctionTable = null;
      for (let j = 0; j < indexedFaces.length; j++) {
        if (indexedFaces[j].person.name === scannedFaces[i].name) {
          face = indexedFaces[j];
          indexedFaces.splice(j, 1);
          break; // region found, stop processing
        }
      }

      if (face == null) {
        faceToInsert.push({
          person: savedPersons.find(
            (p) => p.name === scannedFaces[i].name
          ),
          media: {id: scannedFaces[i].mediaId}
        });
      }
    }
    if (faceToInsert.length > 0) {
      await this.insertChunk(personJunctionTable, faceToInsert, 100);
    }
    await personJunctionTable.remove(indexedFaces, {
      chunk: Math.max(Math.ceil(indexedFaces.length / 500), 1),
    });
  }

  private checkSavingReady(): void {

    if (!this.savingQueue?.length && this.SavingReady) {
      const pr = this.SavingReadyPR;
      this.SavingReady = null;
      this.SavingReadyPR = null;
      this.savingQueue = [];
      if (pr) {
        pr();
      }
    }
  }

  private async saveChunk<T extends ObjectLiteral>(
    repository: Repository<T>,
    entities: T[],
    size: number
  ): Promise<T[]> {
    if (entities.length === 0) {
      return [];
    }
    if (entities.length < size) {
      return await repository.save(entities);
    }
    let list: T[] = [];
    for (let i = 0; i < entities.length / size; i++) {
      list = list.concat(
        await repository.save(entities.slice(i * size, (i + 1) * size))
      );
    }
    return list;
  }

  private async insertChunk<T extends ObjectLiteral>(
    repository: Repository<T>,
    entities: T[],
    size: number
  ): Promise<number[]> {
    if (entities.length === 0) {
      return [];
    }
    if (entities.length < size) {
      return (await repository.insert(entities)).identifiers.map(
        (i: { id: number }) => i.id
      );
    }
    let list: number[] = [];
    for (let i = 0; i < entities.length / size; i++) {
      list = list.concat(
        (
          await repository.insert(entities.slice(i * size, (i + 1) * size))
        ).identifiers.map((ids) => ids['id'])
      );
    }
    return list;
  }
}

1	import {DirectoryBaseDTO, DirectoryDTOUtils, DirectoryPathDTO, ParentDirectoryDTO} from '../../../common/entities/DirectoryDTO';	1✔
2	import {DirectoryEntity} from './enitites/DirectoryEntity';	1✔
3	import {SQLConnection} from './SQLConnection';	1✔
4	import {PhotoEntity, PhotoMetadataEntity} from './enitites/PhotoEntity';	1✔
5	import {Utils} from '../../../common/Utils';	1✔
6	import {PhotoMetadata,} from '../../../common/entities/PhotoDTO';
7	import {Connection, ObjectLiteral, Repository} from 'typeorm';
8	import {MediaEntity} from './enitites/MediaEntity';	1✔
9	import {MediaDTO, MediaDTOUtils} from '../../../common/entities/MediaDTO';	1✔
10	import {VideoEntity} from './enitites/VideoEntity';	1✔
11	import {FileEntity} from './enitites/FileEntity';	1✔
12	import {FileDTO} from '../../../common/entities/FileDTO';
13	import {NotificationManager} from '../NotifocationManager';	1✔
14	import {ObjectManagers} from '../ObjectManagers';	1✔
15	import {Logger} from '../../Logger';	1✔
16	import {ServerPG2ConfMap, ServerSidePG2ConfAction,} from '../../../common/PG2ConfMap';	1✔
17	import {ProjectPath} from '../../ProjectPath';	1✔
18	import * as path from 'path';	1✔
19	import * as fs from 'fs';	1✔
20	import {SearchQueryDTO} from '../../../common/entities/SearchQueryDTO';
21	import {PersonEntry} from './enitites/person/PersonEntry';	1✔
22	import {PersonJunctionTable} from './enitites/person/PersonJunctionTable';	1✔
23	import {MDFileEntity} from './enitites/MDFileEntity';	1✔
24	import {MDFileDTO} from '../../../common/entities/MDFileDTO';
25	import {DiskManager} from '../fileaccess/DiskManager';	1✔
26	import {ProjectedDirectoryCacheEntity} from './enitites/ProjectedDirectoryCacheEntity';	1✔
27
28	const LOG_TAG = '[IndexingManager]';	1✔
29
30	export class IndexingManager {	1✔
31	SavingReady: Promise<void> = null;	317✔
32	private SavingReadyPR: () => void = null;	317✔
33	private savingQueue: { dir: ParentDirectoryDTO; promise: Promise<void>; resolve: () => void; reject: (e: any) => void }[] = [];	317✔
34	private isSaving = false;	317✔
35
36	get IsSavingInProgress(): boolean {
37	return this.SavingReady !== null;	297✔
38	}
39
40	private static async processServerSidePG2Conf(
41	parent: DirectoryPathDTO,
42	files: FileDTO[]
43	): Promise<void> {
44	for (const f of files) {	108✔
45	if (ServerPG2ConfMap[f.name] === ServerSidePG2ConfAction.SAVED_SEARCH) {	12✔
46	const fullMediaPath = path.join(	12✔
47	ProjectPath.ImageFolder,
48	parent.path,
49	parent.name,
50	f.name
51	);
52
53	Logger.silly(	12✔
54	LOG_TAG,
55	'Saving saved-searches to DB from:',
56	fullMediaPath
57	);
58	const savedSearches: { name: string; searchQuery: SearchQueryDTO }[] =
59	JSON.parse(await fs.promises.readFile(fullMediaPath, 'utf8'));	12✔
60	for (const s of savedSearches) {	12✔
61	await ObjectManagers.getInstance().AlbumManager.addIfNotExistSavedSearch(	14✔
62	s.name,
63	s.searchQuery,
64	true
65	);
66	}
67	}
68	}
69	}
70
71	/**
72	* Indexes a dir, but returns early with the scanned version,
73	* does not wait for the DB to be saved
74	*/
75	public async indexDirectory(
76	relativeDirectoryName: string,
77	waitForSave = false	12✔
78	): Promise<ParentDirectoryDTO> {
79	try {	20✔
80	// Check if root is still a valid (non-empty) folder
81	// With weak devices, it is possible that the media that stores
82	// the galley gets unmounted that triggers a full gallery wipe.
83	// Prevent it by stopping indexing on an empty folder.
84	if (fs.readdirSync(ProjectPath.ImageFolder).length === 0) {	20✔
85	throw new Error('Root directory is empty. This is probably error and would erase gallery database. Stopping indexing.');	2✔
86	}
87
88	const scannedDirectory = await DiskManager.scanDirectory(	18✔
89	relativeDirectoryName
90	);
91
92	const dirClone = Utils.clone(scannedDirectory);	18✔
93	// filter server side only config from returning
94	dirClone.metaFile = dirClone.metaFile.filter(	18✔
95	(m) => !ServerPG2ConfMap[m.name]	22✔
96	);
97
98	DirectoryDTOUtils.addReferences(dirClone);	18✔
99
100	if (waitForSave === true) {	18✔
101	// save directory to DB and wait until saving finishes
102	await this.queueForSave(scannedDirectory);	8✔
103	return dirClone;	8✔
104	}
105
106	// save directory to DB in the background
107	this.queueForSave(scannedDirectory).catch(console.error);	10✔
108	return dirClone;	10✔
109	} catch (error) {
110	NotificationManager.warning(	2✔
111	'Unknown indexing error for: ' + relativeDirectoryName,
112	error.toString()
113	);
114	console.error(error);	2✔
115	throw error;	2✔
116	}
117	}
118
119	async resetDB(): Promise<void> {
120	Logger.info(LOG_TAG, 'Resetting DB');	2✔
121	const connection = await SQLConnection.getConnection();	2✔
122	await connection	2✔
123	.getRepository(DirectoryEntity)
124	.createQueryBuilder('directory')
125	.delete()
126	.execute();
127	}
128
129	public async saveToDB(scannedDirectory: ParentDirectoryDTO): Promise<void> {
130	this.isSaving = true;	108✔
131	try {	108✔
132	const connection = await SQLConnection.getConnection();	108✔
133	const serverSideConfigs = scannedDirectory.metaFile.filter(	108✔
134	(m) => !!ServerPG2ConfMap[m.name]	44✔
135	);
136	scannedDirectory.metaFile = scannedDirectory.metaFile.filter(	108✔
137	(m) => !ServerPG2ConfMap[m.name]	44✔
138	);
139	const currentDirId: number = await this.saveParentDir(	108✔
140	connection,
141	scannedDirectory
142	);
143	await this.saveChildDirs(connection, currentDirId, scannedDirectory);	108✔
144	await this.saveMedia(connection, currentDirId, scannedDirectory.media);	108✔
145	await this.saveMetaFiles(connection, currentDirId, scannedDirectory);	108✔
146	await IndexingManager.processServerSidePG2Conf(scannedDirectory, serverSideConfigs);	108✔
147	await ObjectManagers.getInstance().onDataChange(scannedDirectory);	108✔
148	} finally {
149	this.isSaving = false;	108✔
150	}
151	}
152
153	// Todo fix it, once typeorm support connection pools for sqlite
154	/**
155	* Queues up a directory to save to the DB.
156	* Returns a promise that resolves when the directory is saved.
157	*/
158	protected async queueForSave(
159	scannedDirectory: ParentDirectoryDTO
160	): Promise<void> {
161	// Is this dir already queued for saving?
162	const existingIndex = this.savingQueue.findIndex(	24✔
163	(entry): boolean =>
164	entry.dir.name === scannedDirectory.name &&	12!
165	entry.dir.path === scannedDirectory.path &&
166	entry.dir.lastModified === scannedDirectory.lastModified &&
167	entry.dir.lastScanned === scannedDirectory.lastScanned &&
168	(entry.dir.media \|\| entry.dir.media.length) ===	8!
169	(scannedDirectory.media \|\| scannedDirectory.media.length) &&	8!
170	(entry.dir.metaFile \|\| entry.dir.metaFile.length) ===	×
171	(scannedDirectory.metaFile \|\| scannedDirectory.metaFile.length)	×
172	);
173	if (existingIndex !== -1) {	24!
174	return this.savingQueue[existingIndex].promise;	×
175	}
176
177	// queue for saving
178	let resolveFn: () => void;
179	let rejectFn: (e: any) => void;
180	const promise = new Promise<void>((resolve, reject): void => {	24✔
181	resolveFn = resolve;	24✔
182	rejectFn = reject;	24✔
183	});
184
185	this.savingQueue.push({dir: scannedDirectory, promise, resolve: resolveFn, reject: rejectFn});	24✔
186
187	if (this.savingQueue.length > 100) {	24!
188	Logger.warn(LOG_TAG, 'Saving queue is growing large:', this.savingQueue.length);	×
189	}
190
191	this.runSavingLoop().catch(console.error);	24✔
192
193	return promise;	24✔
194	}
195
196	protected async runSavingLoop(): Promise<void> {
197
198	// start saving if not already started
199	if (!this.SavingReady) {	24✔
200	this.SavingReady = new Promise<void>((resolve): void => {	16✔
201	this.SavingReadyPR = resolve;	16✔
202	});
203	}
204	try {	24✔
205	while (this.isSaving === false && this.savingQueue.length > 0) {	24✔
206	const item = this.savingQueue[0];	24✔
207	try {	24✔
208	await this.saveToDB(item.dir);	24✔
209	item.resolve();	24✔
210	} catch (e) {
211	// reject current and remaining queued items to avoid hanging promises
212	item.reject(e);	×
213	this.savingQueue.shift();	×
214	for (const remaining of this.savingQueue) {	×
215	remaining.reject(e);	×
216	}
217	this.savingQueue = [];	×
NEW 218	this.checkSavingReady();	×
UNCOV 219	throw e;	×
220	}
221	this.savingQueue.shift();	24✔
222	}
223	} finally {
224	this.checkSavingReady();	24✔
225	}
226	}
227
228	protected async saveParentDir(
229	connection: Connection,
230	scannedDirectory: ParentDirectoryDTO
231	): Promise<number> {
232	const directoryRepository = connection.getRepository(DirectoryEntity);	108✔
233	const projDirCacheRep = connection.getRepository(ProjectedDirectoryCacheEntity);	108✔
234
235	const currentDir: DirectoryEntity = await directoryRepository	108✔
236	.createQueryBuilder('directory')
237	.where('directory.name = :name AND directory.path = :path', {
238	name: scannedDirectory.name,
239	path: scannedDirectory.path,
240	})
241	.getOne();
242	if (currentDir) {	108✔
243	// Updated parent dir (if it was in the DB previously)
244	currentDir.lastModified = scannedDirectory.lastModified;	20✔
245	currentDir.lastScanned = scannedDirectory.lastScanned;	20✔
246	await directoryRepository.save(currentDir);	20✔
247	return currentDir.id;	20✔
248	} else {
249	return (	88✔
250	await directoryRepository.insert({
251	lastModified: scannedDirectory.lastModified,
252	lastScanned: scannedDirectory.lastScanned,
253	name: scannedDirectory.name,
254	path: scannedDirectory.path,
255	} as DirectoryEntity)
256	).identifiers[0]['id'];
257	}
258	}
259
260	protected async saveChildDirs(
261	connection: Connection,
262	currentDirId: number,
263	scannedDirectory: ParentDirectoryDTO
264	): Promise<void> {
265	const directoryRepository = connection.getRepository(DirectoryEntity);	108✔
266
267	// update subdirectories that does not have a parent
268	await directoryRepository	108✔
269	.createQueryBuilder()
270	.update(DirectoryEntity)
271	.set({parent: currentDirId as unknown})
272	.where('path = :path', {
273	path: DiskManager.pathFromParent(scannedDirectory),
274	})
275	.andWhere('name NOT LIKE :root', {root: DiskManager.dirName('.')})
276	.andWhere('parent IS NULL')
277	.execute();
278
279	// save subdirectories
280	const childDirectories = await directoryRepository	108✔
281	.createQueryBuilder('directory')
282	.leftJoinAndSelect('directory.parent', 'parent')
283	.where('directory.parent = :dir', {
284	dir: currentDirId,
285	})
286	.getMany();
287
288	for (const directory of scannedDirectory.directories) {	108✔
289	// Was this child Dir already indexed before?
290	const dirIndex = childDirectories.findIndex(	184✔
291	(d): boolean => d.name === directory.name	34✔
292	);
293
294	if (dirIndex !== -1) {	184✔
295	// directory found
296	childDirectories.splice(dirIndex, 1);	14✔
297	} else {
298	// dir does not exist yet
299	directory.parent = {id: currentDirId} as ParentDirectoryDTO;	170✔
300	(directory as DirectoryEntity).lastScanned = null; // new child dir, not fully scanned yet	170✔
301	const d = await directoryRepository.insert(	170✔
302	directory as DirectoryEntity
303	);
304
305	await this.saveMedia(	170✔
306	connection,
307	d.identifiers[0]['id'],
308	directory.media
309	);
310	}
311	}
312
313	// Remove child Dirs that are not anymore in the parent dir
314	await directoryRepository.remove(childDirectories, {	108✔
315	chunk: Math.max(Math.ceil(childDirectories.length / 500), 1),
316	});
317	}
318
319	protected async saveMetaFiles(
320	connection: Connection,
321	currentDirID: number,
322	scannedDirectory: ParentDirectoryDTO
323	): Promise<void> {
324	const fileRepository = connection.getRepository(FileEntity);	108✔
325	const MDfileRepository = connection.getRepository(MDFileEntity);	108✔
326	// save files
327	const indexedMetaFiles = await fileRepository	108✔
328	.createQueryBuilder('file')
329	.where('file.directory = :dir', {
330	dir: currentDirID,
331	})
332	.getMany();
333
334	const metaFilesToInsert = [];	108✔
335	const MDFilesToUpdate = [];	108✔
336	for (const item of scannedDirectory.metaFile) {	108✔
337	let metaFile: FileDTO = null;	32✔
338	for (let j = 0; j < indexedMetaFiles.length; j++) {	32✔
339	if (indexedMetaFiles[j].name === item.name) {	4✔
340	metaFile = indexedMetaFiles[j];	4✔
341	indexedMetaFiles.splice(j, 1);	4✔
342	break;	4✔
343	}
344	}
345	if (metaFile == null) {	32✔
346	// not in DB yet
347	item.directory = null;	28✔
348	metaFile = Utils.clone(item);	28✔
349	item.directory = scannedDirectory;	28✔
350	metaFile.directory = {id: currentDirID} as DirectoryBaseDTO;	28✔
351	metaFilesToInsert.push(metaFile);	28✔
352	} else if ((item as MDFileDTO).date) {	4!
353	if ((item as MDFileDTO).date != (metaFile as MDFileDTO).date) {	×
354	(metaFile as MDFileDTO).date = (item as MDFileDTO).date;	×
355	MDFilesToUpdate.push(metaFile);	×
356	}
357	}
358	}
359
360	const MDFiles = metaFilesToInsert.filter(f => !isNaN((f as MDFileDTO).date));	108✔
361	const generalFiles = metaFilesToInsert.filter(f => isNaN((f as MDFileDTO).date));	108✔
362	await fileRepository.save(generalFiles, {	108✔
363	chunk: Math.max(Math.ceil(generalFiles.length / 500), 1),
364	});
365	await MDfileRepository.save(MDFiles, {	108✔
366	chunk: Math.max(Math.ceil(MDFiles.length / 500), 1),
367	});
368	await MDfileRepository.save(MDFilesToUpdate, {	108✔
369	chunk: Math.max(Math.ceil(MDFilesToUpdate.length / 500), 1),
370	});
371	await fileRepository.remove(indexedMetaFiles, {	108✔
372	chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1),
373	});
374	}
375
376	protected async saveMedia(
377	connection: Connection,
378	parentDirId: number,
379	media: MediaDTO[]
380	): Promise<void> {
381	const mediaRepository = connection.getRepository(MediaEntity);	278✔
382	const photoRepository = connection.getRepository(PhotoEntity);	278✔
383	const videoRepository = connection.getRepository(VideoEntity);	278✔
384	// save media
385	let indexedMedia = await mediaRepository	278✔
386	.createQueryBuilder('media')
387	.where('media.directory = :dir', {
388	dir: parentDirId,
389	})
390	.getMany();
391
392	const mediaChange = {	278✔
393	saveP: [] as MediaDTO[], // save/update photo
394	saveV: [] as MediaDTO[], // save/update video
395	insertP: [] as MediaDTO[], // insert photo
396	insertV: [] as MediaDTO[], // insert video
397	};
398	const personsPerPhoto: { faces: { name: string, mediaId?: number }[]; mediaName: string }[] = [];	278✔
399	// eslint-disable-next-line @typescript-eslint/prefer-for-of
400	for (let i = 0; i < media.length; i++) {	278✔
401	let mediaItem: MediaDTO = null;	3,664✔
402	for (let j = 0; j < indexedMedia.length; j++) {	3,664✔
403	if (indexedMedia[j].name === media[i].name) {	156✔
404	mediaItem = indexedMedia[j];	12✔
405	indexedMedia.splice(j, 1);	12✔
406	break;	12✔
407	}
408	}
409
410	const scannedFaces: { name: string }[] = (media[i].metadata as PhotoMetadata).faces \|\| [];	3,664✔
411	if ((media[i].metadata as PhotoMetadata).faces) {	3,664✔
412	// if it has faces, cache them
413	// make the list distinct (some photos may contain the same person multiple times)
414	(media[i].metadata as PhotoMetadataEntity).persons = [	3,336✔
415	...new Set(
416	(media[i].metadata as PhotoMetadata).faces.map((f) => f.name)	6,728✔
417	),
418	];
419	}
420	(media[i].metadata as PhotoMetadataEntity).personsLength = (media[i].metadata as PhotoMetadataEntity)?.persons?.length \|\| 0;	3,664✔
421
422
423	if (mediaItem == null) {	3,664✔
424	// Media not in DB yet
425	media[i].directory = null;	3,652✔
426	mediaItem = Utils.clone(media[i]);	3,652✔
427	mediaItem.directory = {id: parentDirId} as DirectoryBaseDTO;	3,652✔
428	(MediaDTOUtils.isPhoto(mediaItem)	3,652✔
429	? mediaChange.insertP
430	: mediaChange.insertV
431	).push(mediaItem);
432	} else {
433	// Media already in the DB, only needs to be updated
434	delete (mediaItem.metadata as PhotoMetadata).faces;	12✔
435	if (!Utils.equalsFilter(mediaItem.metadata, media[i].metadata)) {	12✔
436	mediaItem.metadata = media[i].metadata;	12✔
437	(MediaDTOUtils.isPhoto(mediaItem)	12!
438	? mediaChange.saveP
439	: mediaChange.saveV
440	).push(mediaItem);
441	}
442	}
443
444	personsPerPhoto.push({	3,664✔
445	faces: scannedFaces,
446	mediaName: mediaItem.name
447	});
448	}
449
450	await this.saveChunk(photoRepository, mediaChange.saveP, 100);	278✔
451	await this.saveChunk(videoRepository, mediaChange.saveV, 100);	278✔
452	await this.saveChunk(photoRepository, mediaChange.insertP, 100);	278✔
453	await this.saveChunk(videoRepository, mediaChange.insertV, 100);	278✔
454
455	indexedMedia = await mediaRepository	278✔
456	.createQueryBuilder('media')
457	.where('media.directory = :dir', {
458	dir: parentDirId,
459	})
460	.select(['media.name', 'media.id'])
461	.getMany();
462
463	const persons: { name: string; mediaId: number }[] = [];	278✔
464	personsPerPhoto.forEach((group): void => {	278✔
465	const mIndex = indexedMedia.findIndex(	3,664✔
466	(m): boolean => m.name === group.mediaName	3,880✔
467	);
468	group.faces.forEach((sf) =>	3,664✔
469	(sf.mediaId = indexedMedia[mIndex].id)	6,728✔
470	);
471
472	persons.push(...group.faces as { name: string; mediaId: number }[]);	3,664✔
473	indexedMedia.splice(mIndex, 1);	3,664✔
474	});
475
476	await this.savePersonsToMedia(connection, parentDirId, persons);	278✔
477	await mediaRepository.remove(indexedMedia);	278✔
478	}
479
480	protected async savePersonsToMedia(
481	connection: Connection,
482	parentDirId: number,
483	scannedFaces: { name: string; mediaId: number }[]
484	): Promise<void> {
485	const personJunctionTable = connection.getRepository(PersonJunctionTable);	278✔
486	const personRepository = connection.getRepository(PersonEntry);	278✔
487
488	const persons: { name: string; mediaId: number }[] = [];	278✔
489
490	// Make a set
491	for (const face of scannedFaces) {	278✔
492	if (persons.findIndex((f) => f.name === face.name) === -1) {	8,998,554✔
493	persons.push(face);	6,596✔
494	}
495	}
496	await ObjectManagers.getInstance().PersonManager.saveAll(persons);	278✔
497	// get saved persons without triggering denormalized data update (i.e.: do not use PersonManager.get).
498	const savedPersons = await personRepository.find();	278✔
499
500	const indexedFaces = await personJunctionTable	278✔
501	.createQueryBuilder('face')
502	.leftJoin('face.media', 'media')
503	.where('media.directory = :directory', {
504	directory: parentDirId,
505	})
506	.leftJoinAndSelect('face.person', 'person')
507	.getMany();
508
509	const faceToInsert: { person: { id: number }, media: { id: number } }[] = [];	278✔
510	// eslint-disable-next-line @typescript-eslint/prefer-for-of
511	for (let i = 0; i < scannedFaces.length; i++) {	278✔
512	// was the Person - media connection already indexed
513	let face: PersonJunctionTable = null;	6,728✔
514	for (let j = 0; j < indexedFaces.length; j++) {	6,728✔
515	if (indexedFaces[j].person.name === scannedFaces[i].name) {	6,990✔
516	face = indexedFaces[j];	6,566✔
517	indexedFaces.splice(j, 1);	6,566✔
518	break; // region found, stop processing	6,566✔
519	}
520	}
521
522	if (face == null) {	6,728✔
523	faceToInsert.push({	162✔
524	person: savedPersons.find(
525	(p) => p.name === scannedFaces[i].name	812✔
526	),
527	media: {id: scannedFaces[i].mediaId}
528	});
529	}
530	}
531	if (faceToInsert.length > 0) {	278✔
532	await this.insertChunk(personJunctionTable, faceToInsert, 100);	34✔
533	}
534	await personJunctionTable.remove(indexedFaces, {	278✔
535	chunk: Math.max(Math.ceil(indexedFaces.length / 500), 1),
536	});
537	}
538
539	private checkSavingReady(): void {
540
541	if (!this.savingQueue?.length && this.SavingReady) {	24✔
542	const pr = this.SavingReadyPR;	16✔
543	this.SavingReady = null;	16✔
544	this.SavingReadyPR = null;	16✔
545	this.savingQueue = [];	16✔
546	if (pr) {	16✔
547	pr();	16✔
548	}
549	}
550	}
551
552	private async saveChunk<T extends ObjectLiteral>(
553	repository: Repository<T>,
554	entities: T[],
555	size: number
556	): Promise<T[]> {
557	if (entities.length === 0) {	1,112✔
558	return [];	806✔
559	}
560	if (entities.length < size) {	306✔
561	return await repository.save(entities);	304✔
562	}
563	let list: T[] = [];	2✔
564	for (let i = 0; i < entities.length / size; i++) {	2✔
565	list = list.concat(	30✔
566	await repository.save(entities.slice(i * size, (i + 1) * size))
567	);
568	}
569	return list;	2✔
570	}
571
572	private async insertChunk<T extends ObjectLiteral>(
573	repository: Repository<T>,
574	entities: T[],
575	size: number
576	): Promise<number[]> {
577	if (entities.length === 0) {	34!
578	return [];	×
579	}
580	if (entities.length < size) {	34✔
581	return (await repository.insert(entities)).identifiers.map(	34✔
582	(i: { id: number }) => i.id	162✔
583	);
584	}
585	let list: number[] = [];	×
586	for (let i = 0; i < entities.length / size; i++) {	×
587	list = list.concat(	×
588	(
589	await repository.insert(entities.slice(i * size, (i + 1) * size))
590	).identifiers.map((ids) => ids['id'])	×
591	);
592	}
593	return list;	×
594	}
595	}

bpatrik / pigallery2 / 19799834564

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous