17685697493

Committed 12 Sep 2025 08:48PM UTC coverage: 65.828% (+1.7%) from 64.102%

Build # 17685697493

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #1030 from bpatrik/db-projection

Implement projected (scoped/filtered) gallery. Fixes #1015

Run Details

1305 of 2243 branches covered (58.18%)

Branch coverage included in aggregate %.

706 of 873 new or added lines in 54 files covered. (80.87%)

16 existing lines in 10 files now uncovered.

4794 of 7022 relevant lines covered (68.27%)

4355.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.04

/src/backend/model/database/IndexingManager.ts

import {DirectoryBaseDTO, DirectoryDTOUtils, DirectoryPathDTO, ParentDirectoryDTO} from '../../../common/entities/DirectoryDTO';
import {DirectoryEntity} from './enitites/DirectoryEntity';
import {SQLConnection} from './SQLConnection';
import {PhotoEntity, PhotoMetadataEntity} from './enitites/PhotoEntity';
import {Utils} from '../../../common/Utils';
import {PhotoMetadata,} from '../../../common/entities/PhotoDTO';
import {Connection, ObjectLiteral, Repository} from 'typeorm';
import {MediaEntity} from './enitites/MediaEntity';
import {MediaDTO, MediaDTOUtils} from '../../../common/entities/MediaDTO';
import {VideoEntity} from './enitites/VideoEntity';
import {FileEntity} from './enitites/FileEntity';
import {FileDTO} from '../../../common/entities/FileDTO';
import {NotificationManager} from '../NotifocationManager';
import {ObjectManagers} from '../ObjectManagers';
import {Logger} from '../../Logger';
import {ServerPG2ConfMap, ServerSidePG2ConfAction,} from '../../../common/PG2ConfMap';
import {ProjectPath} from '../../ProjectPath';
import * as path from 'path';
import * as fs from 'fs';
import {SearchQueryDTO} from '../../../common/entities/SearchQueryDTO';
import {PersonEntry} from './enitites/person/PersonEntry';
import {PersonJunctionTable} from './enitites/person/PersonJunctionTable';
import {MDFileEntity} from './enitites/MDFileEntity';
import {MDFileDTO} from '../../../common/entities/MDFileDTO';
import {DiskManager} from '../fileaccess/DiskManager';
import {ProjectedDirectoryCacheEntity} from './enitites/ProjectedDirectoryCacheEntity';

const LOG_TAG = '[IndexingManager]';

export class IndexingManager {
  SavingReady: Promise<void> = null;
  private SavingReadyPR: () => void = null;
  private savingQueue: { dir: ParentDirectoryDTO; promise: Promise<void>; resolve: () => void; reject: (e: any) => void }[] = [];
  private isSaving = false;

  get IsSavingInProgress(): boolean {
    return this.SavingReady !== null;
  }

  private static async processServerSidePG2Conf(
    parent: DirectoryPathDTO,
    files: FileDTO[]
  ): Promise<void> {
    for (const f of files) {
      if (ServerPG2ConfMap[f.name] === ServerSidePG2ConfAction.SAVED_SEARCH) {
        const fullMediaPath = path.join(
          ProjectPath.ImageFolder,
          parent.path,
          parent.name,
          f.name
        );

        Logger.silly(
          LOG_TAG,
          'Saving saved-searches to DB from:',
          fullMediaPath
        );
        const savedSearches: { name: string; searchQuery: SearchQueryDTO }[] =
          JSON.parse(await fs.promises.readFile(fullMediaPath, 'utf8'));
        for (const s of savedSearches) {
          await ObjectManagers.getInstance().AlbumManager.addIfNotExistSavedSearch(
            s.name,
            s.searchQuery,
            true
          );
        }
      }
    }
  }

  /**
   * Indexes a dir, but returns early with the scanned version,
   * does not wait for the DB to be saved
   */
  public indexDirectory(
    relativeDirectoryName: string,
    waitForSave = false
  ): Promise<ParentDirectoryDTO> {
    // eslint-disable-next-line no-async-promise-executor
    return new Promise(async (resolve, reject): Promise<void> => {
      try {
        // Check if root is still a valid (non-empty) folder
        // With weak devices, it is possible that the media that stores
        // the galley gets unmounted that triggers a full gallery wipe.
        // Prevent it by stopping indexing on an empty folder.
        if (fs.readdirSync(ProjectPath.ImageFolder).length === 0) {
          return reject(new Error('Root directory is empty. This is probably error and would erase gallery database. Stopping indexing.'));
        }

        const scannedDirectory = await DiskManager.scanDirectory(
          relativeDirectoryName
        );


        const dirClone = Utils.clone(scannedDirectory);
        // filter server side only config from returning
        dirClone.metaFile = dirClone.metaFile.filter(
          (m) => !ServerPG2ConfMap[m.name]
        );

        DirectoryDTOUtils.addReferences(dirClone);

        if (waitForSave === true) {
          // save directory to DB and wait until saving finishes
          try {
            await this.queueForSave(scannedDirectory);
          } catch (e) {
            // bubble up save error
            return reject(e);
          }
          return resolve(dirClone);
        }

        // save directory to DB in the background
        resolve(dirClone);
        this.queueForSave(scannedDirectory).catch(console.error);
      } catch (error) {
        NotificationManager.warning(
          'Unknown indexing error for: ' + relativeDirectoryName,
          error.toString()
        );
        console.error(error);
        return reject(error);
      }
    });
  }

  async resetDB(): Promise<void> {
    Logger.info(LOG_TAG, 'Resetting DB');
    const connection = await SQLConnection.getConnection();
    await connection
      .getRepository(DirectoryEntity)
      .createQueryBuilder('directory')
      .delete()
      .execute();
  }

  public async saveToDB(scannedDirectory: ParentDirectoryDTO): Promise<void> {
    this.isSaving = true;
    try {
      const connection = await SQLConnection.getConnection();
      const serverSideConfigs = scannedDirectory.metaFile.filter(
        (m) => !!ServerPG2ConfMap[m.name]
      );
      scannedDirectory.metaFile = scannedDirectory.metaFile.filter(
        (m) => !ServerPG2ConfMap[m.name]
      );
      const currentDirId: number = await this.saveParentDir(
        connection,
        scannedDirectory
      );
      await this.saveChildDirs(connection, currentDirId, scannedDirectory);
      await this.saveMedia(connection, currentDirId, scannedDirectory.media);
      await this.saveMetaFiles(connection, currentDirId, scannedDirectory);
      await IndexingManager.processServerSidePG2Conf(scannedDirectory, serverSideConfigs);
      await ObjectManagers.getInstance().onDataChange(scannedDirectory);
    } finally {
      this.isSaving = false;
    }
  }

  // Todo fix it, once typeorm support connection pools for sqlite
  /**
   * Queues up a directory to save to the DB.
   * Returns a promise that resolves when the directory is saved.
   */
  protected async queueForSave(
    scannedDirectory: ParentDirectoryDTO
  ): Promise<void> {
    // Is this dir already queued for saving?
    const existingIndex = this.savingQueue.findIndex(
      (entry): boolean =>
        entry.dir.name === scannedDirectory.name &&
        entry.dir.path === scannedDirectory.path &&
        entry.dir.lastModified === scannedDirectory.lastModified &&
        entry.dir.lastScanned === scannedDirectory.lastScanned &&
        (entry.dir.media || entry.dir.media.length) ===
        (scannedDirectory.media || scannedDirectory.media.length) &&
        (entry.dir.metaFile || entry.dir.metaFile.length) ===
        (scannedDirectory.metaFile || scannedDirectory.metaFile.length)
    );
    if (existingIndex !== -1) {
      return this.savingQueue[existingIndex].promise;
    }

    // queue for saving
    let resolveFn: () => void;
    let rejectFn: (e: any) => void;
    const promise = new Promise<void>((resolve, reject): void => {
      resolveFn = resolve;
      rejectFn = reject;
    });

    this.savingQueue.push({dir: scannedDirectory, promise, resolve: resolveFn, reject: rejectFn});
    this.runSavingLoop().catch(console.error);

    return promise;
  }

  protected async runSavingLoop(): Promise<void> {

    // start saving if not already started
    if (!this.SavingReady) {
      this.SavingReady = new Promise<void>((resolve): void => {
        this.SavingReadyPR = resolve;
      });
    }
    try {
      while (this.isSaving === false && this.savingQueue.length > 0) {
        const item = this.savingQueue[0];
        try {
          await this.saveToDB(item.dir);
          item.resolve();
        } catch (e) {
          // reject current and remaining queued items to avoid hanging promises
          item.reject(e);
          this.savingQueue.shift();
          for (const remaining of this.savingQueue) {
            remaining.reject(e);
          }
          this.savingQueue = [];
          throw e;
        }
        this.savingQueue.shift();
      }
    } finally {
      if (this.savingQueue.length === 0 && this.SavingReady) {
        const pr = this.SavingReadyPR;
        this.SavingReady = null;
        if (pr) {
          pr();
        }
      }
    }
  }


  protected async saveParentDir(
    connection: Connection,
    scannedDirectory: ParentDirectoryDTO
  ): Promise<number> {
    const directoryRepository = connection.getRepository(DirectoryEntity);
    const projDirCacheRep = connection.getRepository(ProjectedDirectoryCacheEntity);

    const currentDir: DirectoryEntity = await directoryRepository
      .createQueryBuilder('directory')
      .where('directory.name = :name AND directory.path = :path', {
        name: scannedDirectory.name,
        path: scannedDirectory.path,
      })
      .getOne();
    if (currentDir) {
      // Updated parent dir (if it was in the DB previously)
      currentDir.lastModified = scannedDirectory.lastModified;
      currentDir.lastScanned = scannedDirectory.lastScanned;
      await directoryRepository.save(currentDir);
      return currentDir.id;
    } else {
      return (
        await directoryRepository.insert({
          lastModified: scannedDirectory.lastModified,
          lastScanned: scannedDirectory.lastScanned,
          name: scannedDirectory.name,
          path: scannedDirectory.path,
        } as DirectoryEntity)
      ).identifiers[0]['id'];
    }
  }

  protected async saveChildDirs(
    connection: Connection,
    currentDirId: number,
    scannedDirectory: ParentDirectoryDTO
  ): Promise<void> {
    const directoryRepository = connection.getRepository(DirectoryEntity);

    // update subdirectories that does not have a parent
    await directoryRepository
      .createQueryBuilder()
      .update(DirectoryEntity)
      .set({parent: currentDirId as unknown})
      .where('path = :path', {
        path: DiskManager.pathFromParent(scannedDirectory),
      })
      .andWhere('name NOT LIKE :root', {root: DiskManager.dirName('.')})
      .andWhere('parent IS NULL')
      .execute();

    // save subdirectories
    const childDirectories = await directoryRepository
      .createQueryBuilder('directory')
      .leftJoinAndSelect('directory.parent', 'parent')
      .where('directory.parent = :dir', {
        dir: currentDirId,
      })
      .getMany();

    for (const directory of scannedDirectory.directories) {
      // Was this child Dir already indexed before?
      const dirIndex = childDirectories.findIndex(
        (d): boolean => d.name === directory.name
      );

      if (dirIndex !== -1) {
        // directory found
        childDirectories.splice(dirIndex, 1);
      } else {
        // dir does not exist yet
        directory.parent = {id: currentDirId} as ParentDirectoryDTO;
        (directory as DirectoryEntity).lastScanned = null; // new child dir, not fully scanned yet
        const d = await directoryRepository.insert(
          directory as DirectoryEntity
        );

        await this.saveMedia(
          connection,
          d.identifiers[0]['id'],
          directory.media
        );
      }
    }

    // Remove child Dirs that are not anymore in the parent dir
    await directoryRepository.remove(childDirectories, {
      chunk: Math.max(Math.ceil(childDirectories.length / 500), 1),
    });
  }

  protected async saveMetaFiles(
    connection: Connection,
    currentDirID: number,
    scannedDirectory: ParentDirectoryDTO
  ): Promise<void> {
    const fileRepository = connection.getRepository(FileEntity);
    const MDfileRepository = connection.getRepository(MDFileEntity);
    // save files
    const indexedMetaFiles = await fileRepository
      .createQueryBuilder('file')
      .where('file.directory = :dir', {
        dir: currentDirID,
      })
      .getMany();

    const metaFilesToInsert = [];
    const MDFilesToUpdate = [];
    for (const item of scannedDirectory.metaFile) {
      let metaFile: FileDTO = null;
      for (let j = 0; j < indexedMetaFiles.length; j++) {
        if (indexedMetaFiles[j].name === item.name) {
          metaFile = indexedMetaFiles[j];
          indexedMetaFiles.splice(j, 1);
          break;
        }
      }
      if (metaFile == null) {
        // not in DB yet
        item.directory = null;
        metaFile = Utils.clone(item);
        item.directory = scannedDirectory;
        metaFile.directory = {id: currentDirID} as DirectoryBaseDTO;
        metaFilesToInsert.push(metaFile);
      } else if ((item as MDFileDTO).date) {
        if ((item as MDFileDTO).date != (metaFile as MDFileDTO).date) {
          (metaFile as MDFileDTO).date = (item as MDFileDTO).date;
          MDFilesToUpdate.push(metaFile);
        }
      }
    }

    const MDFiles = metaFilesToInsert.filter(f => !isNaN((f as MDFileDTO).date));
    const generalFiles = metaFilesToInsert.filter(f => isNaN((f as MDFileDTO).date));
    await fileRepository.save(generalFiles, {
      chunk: Math.max(Math.ceil(generalFiles.length / 500), 1),
    });
    await MDfileRepository.save(MDFiles, {
      chunk: Math.max(Math.ceil(MDFiles.length / 500), 1),
    });
    await MDfileRepository.save(MDFilesToUpdate, {
      chunk: Math.max(Math.ceil(MDFilesToUpdate.length / 500), 1),
    });
    await fileRepository.remove(indexedMetaFiles, {
      chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1),
    });
  }

  protected async saveMedia(
    connection: Connection,
    parentDirId: number,
    media: MediaDTO[]
  ): Promise<void> {
    const mediaRepository = connection.getRepository(MediaEntity);
    const photoRepository = connection.getRepository(PhotoEntity);
    const videoRepository = connection.getRepository(VideoEntity);
    // save media
    let indexedMedia = await mediaRepository
      .createQueryBuilder('media')
      .where('media.directory = :dir', {
        dir: parentDirId,
      })
      .getMany();

    const mediaChange = {
      saveP: [] as MediaDTO[], // save/update photo
      saveV: [] as MediaDTO[], // save/update video
      insertP: [] as MediaDTO[], // insert photo
      insertV: [] as MediaDTO[], // insert video
    };
    const personsPerPhoto: { faces: { name: string, mediaId?: number }[]; mediaName: string }[] = [];
    // eslint-disable-next-line @typescript-eslint/prefer-for-of
    for (let i = 0; i < media.length; i++) {
      let mediaItem: MediaDTO = null;
      for (let j = 0; j < indexedMedia.length; j++) {
        if (indexedMedia[j].name === media[i].name) {
          mediaItem = indexedMedia[j];
          indexedMedia.splice(j, 1);
          break;
        }
      }

      const scannedFaces: { name: string }[] = (media[i].metadata as PhotoMetadata).faces || [];
      if ((media[i].metadata as PhotoMetadata).faces) {
        // if it has faces, cache them
        // make the list distinct (some photos may contain the same person multiple times)
        (media[i].metadata as PhotoMetadataEntity).persons = [
          ...new Set(
            (media[i].metadata as PhotoMetadata).faces.map((f) => f.name)
          ),
        ];
      }
      (media[i].metadata as PhotoMetadataEntity).personsLength = (media[i].metadata as PhotoMetadataEntity)?.persons?.length || 0;


      if (mediaItem == null) {
        // Media not in DB yet
        media[i].directory = null;
        mediaItem = Utils.clone(media[i]);
        mediaItem.directory = {id: parentDirId} as DirectoryBaseDTO;
        (MediaDTOUtils.isPhoto(mediaItem)
            ? mediaChange.insertP
            : mediaChange.insertV
        ).push(mediaItem);
      } else {
        // Media already in the DB, only needs to be updated
        delete (mediaItem.metadata as PhotoMetadata).faces;
        if (!Utils.equalsFilter(mediaItem.metadata, media[i].metadata)) {
          mediaItem.metadata = media[i].metadata;
          (MediaDTOUtils.isPhoto(mediaItem)
              ? mediaChange.saveP
              : mediaChange.saveV
          ).push(mediaItem);
        }
      }

      personsPerPhoto.push({
        faces: scannedFaces,
        mediaName: mediaItem.name
      });
    }

    await this.saveChunk(photoRepository, mediaChange.saveP, 100);
    await this.saveChunk(videoRepository, mediaChange.saveV, 100);
    await this.saveChunk(photoRepository, mediaChange.insertP, 100);
    await this.saveChunk(videoRepository, mediaChange.insertV, 100);

    indexedMedia = await mediaRepository
      .createQueryBuilder('media')
      .where('media.directory = :dir', {
        dir: parentDirId,
      })
      .select(['media.name', 'media.id'])
      .getMany();

    const persons: { name: string; mediaId: number }[] = [];
    personsPerPhoto.forEach((group): void => {
      const mIndex = indexedMedia.findIndex(
        (m): boolean => m.name === group.mediaName
      );
      group.faces.forEach((sf) =>
        (sf.mediaId = indexedMedia[mIndex].id)
      );

      persons.push(...group.faces as { name: string; mediaId: number }[]);
      indexedMedia.splice(mIndex, 1);
    });

    await this.savePersonsToMedia(connection, parentDirId, persons);
    await mediaRepository.remove(indexedMedia);
  }

  protected async savePersonsToMedia(
    connection: Connection,
    parentDirId: number,
    scannedFaces: { name: string; mediaId: number }[]
  ): Promise<void> {
    const personJunctionTable = connection.getRepository(PersonJunctionTable);
    const personRepository = connection.getRepository(PersonEntry);

    const persons: { name: string; mediaId: number }[] = [];

    // Make a set
    for (const face of scannedFaces) {
      if (persons.findIndex((f) => f.name === face.name) === -1) {
        persons.push(face);
      }
    }
    await ObjectManagers.getInstance().PersonManager.saveAll(persons);
    // get saved persons without triggering denormalized data update (i.e.: do not use PersonManager.get).
    const savedPersons = await personRepository.find();

    const indexedFaces = await personJunctionTable
      .createQueryBuilder('face')
      .leftJoin('face.media', 'media')
      .where('media.directory = :directory', {
        directory: parentDirId,
      })
      .leftJoinAndSelect('face.person', 'person')
      .getMany();

    const faceToInsert: { person: { id: number }, media: { id: number } }[] = [];
    // eslint-disable-next-line @typescript-eslint/prefer-for-of
    for (let i = 0; i < scannedFaces.length; i++) {
      // was the Person - media connection already indexed
      let face: PersonJunctionTable = null;
      for (let j = 0; j < indexedFaces.length; j++) {
        if (indexedFaces[j].person.name === scannedFaces[i].name) {
          face = indexedFaces[j];
          indexedFaces.splice(j, 1);
          break; // region found, stop processing
        }
      }

      if (face == null) {
        faceToInsert.push({
          person: savedPersons.find(
            (p) => p.name === scannedFaces[i].name
          ),
          media: {id: scannedFaces[i].mediaId}
        });
      }
    }
    if (faceToInsert.length > 0) {
      await this.insertChunk(personJunctionTable, faceToInsert, 100);
    }
    await personJunctionTable.remove(indexedFaces, {
      chunk: Math.max(Math.ceil(indexedFaces.length / 500), 1),
    });
  }

  private async saveChunk<T extends ObjectLiteral>(
    repository: Repository<T>,
    entities: T[],
    size: number
  ): Promise<T[]> {
    if (entities.length === 0) {
      return [];
    }
    if (entities.length < size) {
      return await repository.save(entities);
    }
    let list: T[] = [];
    for (let i = 0; i < entities.length / size; i++) {
      list = list.concat(
        await repository.save(entities.slice(i * size, (i + 1) * size))
      );
    }
    return list;
  }

  private async insertChunk<T extends ObjectLiteral>(
    repository: Repository<T>,
    entities: T[],
    size: number
  ): Promise<number[]> {
    if (entities.length === 0) {
      return [];
    }
    if (entities.length < size) {
      return (await repository.insert(entities)).identifiers.map(
        (i: { id: number }) => i.id
      );
    }
    let list: number[] = [];
    for (let i = 0; i < entities.length / size; i++) {
      list = list.concat(
        (
          await repository.insert(entities.slice(i * size, (i + 1) * size))
        ).identifiers.map((ids) => ids['id'])
      );
    }
    return list;
  }
}

1	import {DirectoryBaseDTO, DirectoryDTOUtils, DirectoryPathDTO, ParentDirectoryDTO} from '../../../common/entities/DirectoryDTO';	1✔
2	import {DirectoryEntity} from './enitites/DirectoryEntity';	1✔
3	import {SQLConnection} from './SQLConnection';	1✔
4	import {PhotoEntity, PhotoMetadataEntity} from './enitites/PhotoEntity';	1✔
5	import {Utils} from '../../../common/Utils';	1✔
6	import {PhotoMetadata,} from '../../../common/entities/PhotoDTO';
7	import {Connection, ObjectLiteral, Repository} from 'typeorm';
8	import {MediaEntity} from './enitites/MediaEntity';	1✔
9	import {MediaDTO, MediaDTOUtils} from '../../../common/entities/MediaDTO';	1✔
10	import {VideoEntity} from './enitites/VideoEntity';	1✔
11	import {FileEntity} from './enitites/FileEntity';	1✔
12	import {FileDTO} from '../../../common/entities/FileDTO';
13	import {NotificationManager} from '../NotifocationManager';	1✔
14	import {ObjectManagers} from '../ObjectManagers';	1✔
15	import {Logger} from '../../Logger';	1✔
16	import {ServerPG2ConfMap, ServerSidePG2ConfAction,} from '../../../common/PG2ConfMap';	1✔
17	import {ProjectPath} from '../../ProjectPath';	1✔
18	import * as path from 'path';	1✔
19	import * as fs from 'fs';	1✔
20	import {SearchQueryDTO} from '../../../common/entities/SearchQueryDTO';
21	import {PersonEntry} from './enitites/person/PersonEntry';	1✔
22	import {PersonJunctionTable} from './enitites/person/PersonJunctionTable';	1✔
23	import {MDFileEntity} from './enitites/MDFileEntity';	1✔
24	import {MDFileDTO} from '../../../common/entities/MDFileDTO';
25	import {DiskManager} from '../fileaccess/DiskManager';	1✔
26	import {ProjectedDirectoryCacheEntity} from './enitites/ProjectedDirectoryCacheEntity';	1✔
27
28	const LOG_TAG = '[IndexingManager]';	1✔
29
30	export class IndexingManager {	1✔
31	SavingReady: Promise<void> = null;	281✔
32	private SavingReadyPR: () => void = null;	281✔
33	private savingQueue: { dir: ParentDirectoryDTO; promise: Promise<void>; resolve: () => void; reject: (e: any) => void }[] = [];	281✔
34	private isSaving = false;	281✔
35
36	get IsSavingInProgress(): boolean {
37	return this.SavingReady !== null;	257✔
38	}
39
40	private static async processServerSidePG2Conf(
41	parent: DirectoryPathDTO,
42	files: FileDTO[]
43	): Promise<void> {
44	for (const f of files) {	98✔
45	if (ServerPG2ConfMap[f.name] === ServerSidePG2ConfAction.SAVED_SEARCH) {	8✔
46	const fullMediaPath = path.join(	8✔
47	ProjectPath.ImageFolder,
48	parent.path,
49	parent.name,
50	f.name
51	);
52
53	Logger.silly(	8✔
54	LOG_TAG,
55	'Saving saved-searches to DB from:',
56	fullMediaPath
57	);
58	const savedSearches: { name: string; searchQuery: SearchQueryDTO }[] =
59	JSON.parse(await fs.promises.readFile(fullMediaPath, 'utf8'));	8✔
60	for (const s of savedSearches) {	8✔
61	await ObjectManagers.getInstance().AlbumManager.addIfNotExistSavedSearch(	8✔
62	s.name,
63	s.searchQuery,
64	true
65	);
66	}
67	}
68	}
69	}
70
71	/**
72	* Indexes a dir, but returns early with the scanned version,
73	* does not wait for the DB to be saved
74	*/
75	public indexDirectory(
76	relativeDirectoryName: string,
77	waitForSave = false	8✔
78	): Promise<ParentDirectoryDTO> {
79	// eslint-disable-next-line no-async-promise-executor
80	return new Promise(async (resolve, reject): Promise<void> => {	14✔
81	try {	14✔
82	// Check if root is still a valid (non-empty) folder
83	// With weak devices, it is possible that the media that stores
84	// the galley gets unmounted that triggers a full gallery wipe.
85	// Prevent it by stopping indexing on an empty folder.
86	if (fs.readdirSync(ProjectPath.ImageFolder).length === 0) {	14✔
87	return reject(new Error('Root directory is empty. This is probably error and would erase gallery database. Stopping indexing.'));	2✔
88	}
89
90	const scannedDirectory = await DiskManager.scanDirectory(	12✔
91	relativeDirectoryName
92	);
93
94
95	const dirClone = Utils.clone(scannedDirectory);	12✔
96	// filter server side only config from returning
97	dirClone.metaFile = dirClone.metaFile.filter(	12✔
98	(m) => !ServerPG2ConfMap[m.name]	8✔
99	);
100
101	DirectoryDTOUtils.addReferences(dirClone);	12✔
102
103	if (waitForSave === true) {	12✔
104	// save directory to DB and wait until saving finishes
105	try {	6✔
106	await this.queueForSave(scannedDirectory);	6✔
107	} catch (e) {
108	// bubble up save error
NEW 109	return reject(e);	×
110	}
111	return resolve(dirClone);	6✔
112	}
113
114	// save directory to DB in the background
115	resolve(dirClone);	6✔
116	this.queueForSave(scannedDirectory).catch(console.error);	6✔
117	} catch (error) {
118	NotificationManager.warning(	×
119	'Unknown indexing error for: ' + relativeDirectoryName,
120	error.toString()
121	);
122	console.error(error);	×
123	return reject(error);	×
124	}
125	});
126	}
127
128	async resetDB(): Promise<void> {
129	Logger.info(LOG_TAG, 'Resetting DB');	2✔
130	const connection = await SQLConnection.getConnection();	2✔
131	await connection	2✔
132	.getRepository(DirectoryEntity)
133	.createQueryBuilder('directory')
134	.delete()
135	.execute();
136	}
137
138	public async saveToDB(scannedDirectory: ParentDirectoryDTO): Promise<void> {
139	this.isSaving = true;	98✔
140	try {	98✔
141	const connection = await SQLConnection.getConnection();	98✔
142	const serverSideConfigs = scannedDirectory.metaFile.filter(	98✔
143	(m) => !!ServerPG2ConfMap[m.name]	30✔
144	);
145	scannedDirectory.metaFile = scannedDirectory.metaFile.filter(	98✔
146	(m) => !ServerPG2ConfMap[m.name]	30✔
147	);
148	const currentDirId: number = await this.saveParentDir(	98✔
149	connection,
150	scannedDirectory
151	);
152	await this.saveChildDirs(connection, currentDirId, scannedDirectory);	98✔
153	await this.saveMedia(connection, currentDirId, scannedDirectory.media);	98✔
154	await this.saveMetaFiles(connection, currentDirId, scannedDirectory);	98✔
155	await IndexingManager.processServerSidePG2Conf(scannedDirectory, serverSideConfigs);	98✔
156	await ObjectManagers.getInstance().onDataChange(scannedDirectory);	98✔
157	} finally {
158	this.isSaving = false;	98✔
159	}
160	}
161
162	// Todo fix it, once typeorm support connection pools for sqlite
163	/**
164	* Queues up a directory to save to the DB.
165	* Returns a promise that resolves when the directory is saved.
166	*/
167	protected async queueForSave(
168	scannedDirectory: ParentDirectoryDTO
169	): Promise<void> {
170	// Is this dir already queued for saving?
171	const existingIndex = this.savingQueue.findIndex(	18✔
172	(entry): boolean =>
173	entry.dir.name === scannedDirectory.name &&	12!
174	entry.dir.path === scannedDirectory.path &&
175	entry.dir.lastModified === scannedDirectory.lastModified &&
176	entry.dir.lastScanned === scannedDirectory.lastScanned &&
177	(entry.dir.media \|\| entry.dir.media.length) ===	8!
178	(scannedDirectory.media \|\| scannedDirectory.media.length) &&	8!
179	(entry.dir.metaFile \|\| entry.dir.metaFile.length) ===	×
180	(scannedDirectory.metaFile \|\| scannedDirectory.metaFile.length)	×
181	);
182	if (existingIndex !== -1) {	18!
NEW 183	return this.savingQueue[existingIndex].promise;	×
184	}
185
186	// queue for saving
187	let resolveFn: () => void;
188	let rejectFn: (e: any) => void;
189	const promise = new Promise<void>((resolve, reject): void => {	18✔
190	resolveFn = resolve;	18✔
191	rejectFn = reject;	18✔
192	});
193
194	this.savingQueue.push({dir: scannedDirectory, promise, resolve: resolveFn, reject: rejectFn});	18✔
195	this.runSavingLoop().catch(console.error);	18✔
196
197	return promise;	18✔
198	}
199
200	protected async runSavingLoop(): Promise<void> {
201
202	// start saving if not already started
203	if (!this.SavingReady) {	18✔
204	this.SavingReady = new Promise<void>((resolve): void => {	10✔
205	this.SavingReadyPR = resolve;	10✔
206	});
207	}
208	try {	18✔
209	while (this.isSaving === false && this.savingQueue.length > 0) {	18✔
210	const item = this.savingQueue[0];	18✔
211	try {	18✔
212	await this.saveToDB(item.dir);	18✔
213	item.resolve();	18✔
214	} catch (e) {
215	// reject current and remaining queued items to avoid hanging promises
NEW 216	item.reject(e);	×
NEW 217	this.savingQueue.shift();	×
NEW 218	for (const remaining of this.savingQueue) {	×
NEW 219	remaining.reject(e);	×
220	}
NEW 221	this.savingQueue = [];	×
NEW 222	throw e;	×
223	}
224	this.savingQueue.shift();	18✔
225	}
226	} finally {
227	if (this.savingQueue.length === 0 && this.SavingReady) {	18✔
228	const pr = this.SavingReadyPR;	10✔
229	this.SavingReady = null;	10✔
230	if (pr) {	10✔
231	pr();	10✔
232	}
233	}
234	}
235	}
236
237
238	protected async saveParentDir(
239	connection: Connection,
240	scannedDirectory: ParentDirectoryDTO
241	): Promise<number> {
242	const directoryRepository = connection.getRepository(DirectoryEntity);	98✔
243	const projDirCacheRep = connection.getRepository(ProjectedDirectoryCacheEntity);	98✔
244
245	const currentDir: DirectoryEntity = await directoryRepository	98✔
246	.createQueryBuilder('directory')
247	.where('directory.name = :name AND directory.path = :path', {
248	name: scannedDirectory.name,
249	path: scannedDirectory.path,
250	})
251	.getOne();
252	if (currentDir) {	98✔
253	// Updated parent dir (if it was in the DB previously)
254	currentDir.lastModified = scannedDirectory.lastModified;	18✔
255	currentDir.lastScanned = scannedDirectory.lastScanned;	18✔
256	await directoryRepository.save(currentDir);	18✔
257	return currentDir.id;	18✔
258	} else {
259	return (	80✔
260	await directoryRepository.insert({
261	lastModified: scannedDirectory.lastModified,
262	lastScanned: scannedDirectory.lastScanned,
263	name: scannedDirectory.name,
264	path: scannedDirectory.path,
265	} as DirectoryEntity)
266	).identifiers[0]['id'];
267	}
268	}
269
270	protected async saveChildDirs(
271	connection: Connection,
272	currentDirId: number,
273	scannedDirectory: ParentDirectoryDTO
274	): Promise<void> {
275	const directoryRepository = connection.getRepository(DirectoryEntity);	98✔
276
277	// update subdirectories that does not have a parent
278	await directoryRepository	98✔
279	.createQueryBuilder()
280	.update(DirectoryEntity)
281	.set({parent: currentDirId as unknown})
282	.where('path = :path', {
283	path: DiskManager.pathFromParent(scannedDirectory),
284	})
285	.andWhere('name NOT LIKE :root', {root: DiskManager.dirName('.')})
286	.andWhere('parent IS NULL')
287	.execute();
288
289	// save subdirectories
290	const childDirectories = await directoryRepository	98✔
291	.createQueryBuilder('directory')
292	.leftJoinAndSelect('directory.parent', 'parent')
293	.where('directory.parent = :dir', {
294	dir: currentDirId,
295	})
296	.getMany();
297
298	for (const directory of scannedDirectory.directories) {	98✔
299	// Was this child Dir already indexed before?
300	const dirIndex = childDirectories.findIndex(	152✔
301	(d): boolean => d.name === directory.name	14✔
302	);
303
304	if (dirIndex !== -1) {	152✔
305	// directory found
306	childDirectories.splice(dirIndex, 1);	14✔
307	} else {
308	// dir does not exist yet
309	directory.parent = {id: currentDirId} as ParentDirectoryDTO;	138✔
310	(directory as DirectoryEntity).lastScanned = null; // new child dir, not fully scanned yet	138✔
311	const d = await directoryRepository.insert(	138✔
312	directory as DirectoryEntity
313	);
314
315	await this.saveMedia(	138✔
316	connection,
317	d.identifiers[0]['id'],
318	directory.media
319	);
320	}
321	}
322
323	// Remove child Dirs that are not anymore in the parent dir
324	await directoryRepository.remove(childDirectories, {	98✔
325	chunk: Math.max(Math.ceil(childDirectories.length / 500), 1),
326	});
327	}
328
329	protected async saveMetaFiles(
330	connection: Connection,
331	currentDirID: number,
332	scannedDirectory: ParentDirectoryDTO
333	): Promise<void> {
334	const fileRepository = connection.getRepository(FileEntity);	98✔
335	const MDfileRepository = connection.getRepository(MDFileEntity);	98✔
336	// save files
337	const indexedMetaFiles = await fileRepository	98✔
338	.createQueryBuilder('file')
339	.where('file.directory = :dir', {
340	dir: currentDirID,
341	})
342	.getMany();
343
344	const metaFilesToInsert = [];	98✔
345	const MDFilesToUpdate = [];	98✔
346	for (const item of scannedDirectory.metaFile) {	98✔
347	let metaFile: FileDTO = null;	22✔
348	for (let j = 0; j < indexedMetaFiles.length; j++) {	22✔
349	if (indexedMetaFiles[j].name === item.name) {	4✔
350	metaFile = indexedMetaFiles[j];	4✔
351	indexedMetaFiles.splice(j, 1);	4✔
352	break;	4✔
353	}
354	}
355	if (metaFile == null) {	22✔
356	// not in DB yet
357	item.directory = null;	18✔
358	metaFile = Utils.clone(item);	18✔
359	item.directory = scannedDirectory;	18✔
360	metaFile.directory = {id: currentDirID} as DirectoryBaseDTO;	18✔
361	metaFilesToInsert.push(metaFile);	18✔
362	} else if ((item as MDFileDTO).date) {	4!
363	if ((item as MDFileDTO).date != (metaFile as MDFileDTO).date) {	×
364	(metaFile as MDFileDTO).date = (item as MDFileDTO).date;	×
365	MDFilesToUpdate.push(metaFile);	×
366	}
367	}
368	}
369
370	const MDFiles = metaFilesToInsert.filter(f => !isNaN((f as MDFileDTO).date));	98✔
371	const generalFiles = metaFilesToInsert.filter(f => isNaN((f as MDFileDTO).date));	98✔
372	await fileRepository.save(generalFiles, {	98✔
373	chunk: Math.max(Math.ceil(generalFiles.length / 500), 1),
374	});
375	await MDfileRepository.save(MDFiles, {	98✔
376	chunk: Math.max(Math.ceil(MDFiles.length / 500), 1),
377	});
378	await MDfileRepository.save(MDFilesToUpdate, {	98✔
379	chunk: Math.max(Math.ceil(MDFilesToUpdate.length / 500), 1),
380	});
381	await fileRepository.remove(indexedMetaFiles, {	98✔
382	chunk: Math.max(Math.ceil(indexedMetaFiles.length / 500), 1),
383	});
384	}
385
386	protected async saveMedia(
387	connection: Connection,
388	parentDirId: number,
389	media: MediaDTO[]
390	): Promise<void> {
391	const mediaRepository = connection.getRepository(MediaEntity);	236✔
392	const photoRepository = connection.getRepository(PhotoEntity);	236✔
393	const videoRepository = connection.getRepository(VideoEntity);	236✔
394	// save media
395	let indexedMedia = await mediaRepository	236✔
396	.createQueryBuilder('media')
397	.where('media.directory = :dir', {
398	dir: parentDirId,
399	})
400	.getMany();
401
402	const mediaChange = {	236✔
403	saveP: [] as MediaDTO[], // save/update photo
404	saveV: [] as MediaDTO[], // save/update video
405	insertP: [] as MediaDTO[], // insert photo
406	insertV: [] as MediaDTO[], // insert video
407	};
408	const personsPerPhoto: { faces: { name: string, mediaId?: number }[]; mediaName: string }[] = [];	236✔
409	// eslint-disable-next-line @typescript-eslint/prefer-for-of
410	for (let i = 0; i < media.length; i++) {	236✔
411	let mediaItem: MediaDTO = null;	3,486✔
412	for (let j = 0; j < indexedMedia.length; j++) {	3,486✔
413	if (indexedMedia[j].name === media[i].name) {	12✔
414	mediaItem = indexedMedia[j];	12✔
415	indexedMedia.splice(j, 1);	12✔
416	break;	12✔
417	}
418	}
419
420	const scannedFaces: { name: string }[] = (media[i].metadata as PhotoMetadata).faces \|\| [];	3,486✔
421	if ((media[i].metadata as PhotoMetadata).faces) {	3,486✔
422	// if it has faces, cache them
423	// make the list distinct (some photos may contain the same person multiple times)
424	(media[i].metadata as PhotoMetadataEntity).persons = [	3,258✔
425	...new Set(
426	(media[i].metadata as PhotoMetadata).faces.map((f) => f.name)	6,592✔
427	),
428	];
429	}
430	(media[i].metadata as PhotoMetadataEntity).personsLength = (media[i].metadata as PhotoMetadataEntity)?.persons?.length \|\| 0;	3,486✔
431
432
433	if (mediaItem == null) {	3,486✔
434	// Media not in DB yet
435	media[i].directory = null;	3,474✔
436	mediaItem = Utils.clone(media[i]);	3,474✔
437	mediaItem.directory = {id: parentDirId} as DirectoryBaseDTO;	3,474✔
438	(MediaDTOUtils.isPhoto(mediaItem)	3,474✔
439	? mediaChange.insertP
440	: mediaChange.insertV
441	).push(mediaItem);
442	} else {
443	// Media already in the DB, only needs to be updated
444	delete (mediaItem.metadata as PhotoMetadata).faces;	12✔
445	if (!Utils.equalsFilter(mediaItem.metadata, media[i].metadata)) {	12✔
446	mediaItem.metadata = media[i].metadata;	12✔
447	(MediaDTOUtils.isPhoto(mediaItem)	12!
448	? mediaChange.saveP
449	: mediaChange.saveV
450	).push(mediaItem);
451	}
452	}
453
454	personsPerPhoto.push({	3,486✔
455	faces: scannedFaces,
456	mediaName: mediaItem.name
457	});
458	}
459
460	await this.saveChunk(photoRepository, mediaChange.saveP, 100);	236✔
461	await this.saveChunk(videoRepository, mediaChange.saveV, 100);	236✔
462	await this.saveChunk(photoRepository, mediaChange.insertP, 100);	236✔
463	await this.saveChunk(videoRepository, mediaChange.insertV, 100);	236✔
464
465	indexedMedia = await mediaRepository	236✔
466	.createQueryBuilder('media')
467	.where('media.directory = :dir', {
468	dir: parentDirId,
469	})
470	.select(['media.name', 'media.id'])
471	.getMany();
472
473	const persons: { name: string; mediaId: number }[] = [];	236✔
474	personsPerPhoto.forEach((group): void => {	236✔
475	const mIndex = indexedMedia.findIndex(	3,486✔
476	(m): boolean => m.name === group.mediaName	3,540✔
477	);
478	group.faces.forEach((sf) =>	3,486✔
479	(sf.mediaId = indexedMedia[mIndex].id)	6,592✔
480	);
481
482	persons.push(...group.faces as { name: string; mediaId: number }[]);	3,486✔
483	indexedMedia.splice(mIndex, 1);	3,486✔
484	});
485
486	await this.savePersonsToMedia(connection, parentDirId, persons);	236✔
487	await mediaRepository.remove(indexedMedia);	236✔
488	}
489
490	protected async savePersonsToMedia(
491	connection: Connection,
492	parentDirId: number,
493	scannedFaces: { name: string; mediaId: number }[]
494	): Promise<void> {
495	const personJunctionTable = connection.getRepository(PersonJunctionTable);	236✔
496	const personRepository = connection.getRepository(PersonEntry);	236✔
497
498	const persons: { name: string; mediaId: number }[] = [];	236✔
499
500	// Make a set
501	for (const face of scannedFaces) {	236✔
502	if (persons.findIndex((f) => f.name === face.name) === -1) {	8,998,180✔
503	persons.push(face);	6,516✔
504	}
505	}
506	await ObjectManagers.getInstance().PersonManager.saveAll(persons);	236✔
507	// get saved persons without triggering denormalized data update (i.e.: do not use PersonManager.get).
508	const savedPersons = await personRepository.find();	236✔
509
510	const indexedFaces = await personJunctionTable	236✔
511	.createQueryBuilder('face')
512	.leftJoin('face.media', 'media')
513	.where('media.directory = :directory', {
514	directory: parentDirId,
515	})
516	.leftJoinAndSelect('face.person', 'person')
517	.getMany();
518
519	const faceToInsert: { person: { id: number }, media: { id: number } }[] = [];	236✔
520	// eslint-disable-next-line @typescript-eslint/prefer-for-of
521	for (let i = 0; i < scannedFaces.length; i++) {	236✔
522	// was the Person - media connection already indexed
523	let face: PersonJunctionTable = null;	6,592✔
524	for (let j = 0; j < indexedFaces.length; j++) {	6,592✔
525	if (indexedFaces[j].person.name === scannedFaces[i].name) {	6,650✔
526	face = indexedFaces[j];	6,486✔
527	indexedFaces.splice(j, 1);	6,486✔
528	break; // region found, stop processing	6,486✔
529	}
530	}
531
532	if (face == null) {	6,592✔
533	faceToInsert.push({	106✔
534	person: savedPersons.find(
535	(p) => p.name === scannedFaces[i].name	500✔
536	),
537	media: {id: scannedFaces[i].mediaId}
538	});
539	}
540	}
541	if (faceToInsert.length > 0) {	236✔
542	await this.insertChunk(personJunctionTable, faceToInsert, 100);	28✔
543	}
544	await personJunctionTable.remove(indexedFaces, {	236✔
545	chunk: Math.max(Math.ceil(indexedFaces.length / 500), 1),
546	});
547	}
548
549	private async saveChunk<T extends ObjectLiteral>(
550	repository: Repository<T>,
551	entities: T[],
552	size: number
553	): Promise<T[]> {
554	if (entities.length === 0) {	944✔
555	return [];	686✔
556	}
557	if (entities.length < size) {	258✔
558	return await repository.save(entities);	256✔
559	}
560	let list: T[] = [];	2✔
561	for (let i = 0; i < entities.length / size; i++) {	2✔
562	list = list.concat(	30✔
563	await repository.save(entities.slice(i * size, (i + 1) * size))
564	);
565	}
566	return list;	2✔
567	}
568
569	private async insertChunk<T extends ObjectLiteral>(
570	repository: Repository<T>,
571	entities: T[],
572	size: number
573	): Promise<number[]> {
574	if (entities.length === 0) {	28!
575	return [];	×
576	}
577	if (entities.length < size) {	28✔
578	return (await repository.insert(entities)).identifiers.map(	28✔
579	(i: { id: number }) => i.id	106✔
580	);
581	}
582	let list: number[] = [];	×
583	for (let i = 0; i < entities.length / size; i++) {	×
584	list = list.concat(	×
585	(
586	await repository.insert(entities.slice(i * size, (i + 1) * size))
587	).identifiers.map((ids) => ids['id'])	×
588	);
589	}
590	return list;	×
591	}
592	}

bpatrik / pigallery2 / 17685697493

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous