Lint fixes
This commit is contained in:
parent
eefac7fd01
commit
18937581a5
4 changed files with 44 additions and 43 deletions
|
@ -13,11 +13,14 @@ import type {
|
|||
PutEmbeddingRequest,
|
||||
} from "types/embedding";
|
||||
import { EnteFile } from "types/file";
|
||||
import { getLatestVersionEmbeddings,getLatestVersionFileEmbeddings } from "utils/embedding";
|
||||
import {
|
||||
getLatestVersionEmbeddings,
|
||||
getLatestVersionFileEmbeddings,
|
||||
} from "utils/embedding";
|
||||
import { FileML } from "utils/machineLearning/mldataMappers";
|
||||
import { getLocalCollections } from "./collectionService";
|
||||
import { getAllLocalFiles } from "./fileService";
|
||||
import { getLocalTrashedFiles } from "./trashService";
|
||||
import { FileML } from "utils/machineLearning/mldataMappers";
|
||||
|
||||
const ENDPOINT = getEndpoint();
|
||||
|
||||
|
@ -154,7 +157,7 @@ export const syncEmbeddings = async () => {
|
|||
export const syncFileEmbeddings = async () => {
|
||||
const models: EmbeddingModel[] = ["file-ml-clip-face"];
|
||||
try {
|
||||
let allEmbeddings :FileML[] = await getFileMLEmbeddings();
|
||||
let allEmbeddings: FileML[] = await getFileMLEmbeddings();
|
||||
const localFiles = await getAllLocalFiles();
|
||||
const hiddenAlbums = await getLocalCollections("hidden");
|
||||
const localTrashFiles = await getLocalTrashedFiles();
|
||||
|
@ -179,7 +182,6 @@ export const syncFileEmbeddings = async () => {
|
|||
const newEmbeddings = await Promise.all(
|
||||
response.diff.map(async (embedding) => {
|
||||
try {
|
||||
|
||||
const worker =
|
||||
await ComlinkCryptoWorker.getInstance();
|
||||
const fileKey = fileIdToKeyMap.get(
|
||||
|
@ -196,7 +198,7 @@ export const syncFileEmbeddings = async () => {
|
|||
|
||||
return {
|
||||
...decryptedData,
|
||||
updatedAt: embedding.updatedAt
|
||||
updatedAt: embedding.updatedAt,
|
||||
} as unknown as FileML;
|
||||
} catch (e) {
|
||||
let hasHiddenAlbums = false;
|
||||
|
@ -229,7 +231,6 @@ export const syncFileEmbeddings = async () => {
|
|||
}
|
||||
};
|
||||
|
||||
|
||||
export const getEmbeddingsDiff = async (
|
||||
sinceTime: number,
|
||||
model: EmbeddingModel,
|
||||
|
@ -263,7 +264,7 @@ export const putEmbedding = async (
|
|||
try {
|
||||
const token = getToken();
|
||||
if (!token) {
|
||||
log.info('putEmbedding failed: token not found');
|
||||
log.info("putEmbedding failed: token not found");
|
||||
throw Error(CustomError.TOKEN_MISSING);
|
||||
}
|
||||
const resp = await HTTPService.put(
|
||||
|
|
|
@ -1,12 +1,13 @@
|
|||
import log from "@/next/log";
|
||||
import { APPS } from "@ente/shared/apps/constants";
|
||||
import { CustomError, parseUploadErrorCodes } from "@ente/shared/error";
|
||||
import ComlinkCryptoWorker from "@ente/shared/crypto";
|
||||
import { CustomError, parseUploadErrorCodes } from "@ente/shared/error";
|
||||
import "@tensorflow/tfjs-backend-cpu";
|
||||
import "@tensorflow/tfjs-backend-webgl";
|
||||
import * as tf from "@tensorflow/tfjs-core";
|
||||
import { MAX_ML_SYNC_ERROR_COUNT } from "constants/mlConfig";
|
||||
import downloadManager from "services/download";
|
||||
import { putEmbedding } from "services/embeddingService";
|
||||
import { getLocalFiles } from "services/fileService";
|
||||
import { EnteFile } from "types/file";
|
||||
import {
|
||||
|
@ -16,14 +17,13 @@ import {
|
|||
MlFileData,
|
||||
} from "types/machineLearning";
|
||||
import { getMLSyncConfig } from "utils/machineLearning/config";
|
||||
import { LocalFileMlDataToServerFileMl } from "utils/machineLearning/mldataMappers";
|
||||
import mlIDbStorage from "utils/storage/mlIDbStorage";
|
||||
import FaceService from "./faceService";
|
||||
import { MLFactory } from "./machineLearningFactory";
|
||||
import ObjectService from "./objectService";
|
||||
import PeopleService from "./peopleService";
|
||||
import ReaderService from "./readerService";
|
||||
import { LocalFileMlDataToServerFileMl } from "utils/machineLearning/mldataMappers";
|
||||
import { putEmbedding } from "services/embeddingService";
|
||||
|
||||
class MachineLearningService {
|
||||
private initialized = false;
|
||||
|
@ -165,7 +165,7 @@ class MachineLearningService {
|
|||
|
||||
log.info("syncLocalFiles", Date.now() - startTime, "ms");
|
||||
}
|
||||
|
||||
|
||||
private async getOutOfSyncFiles(syncContext: MLSyncContext) {
|
||||
const startTime = Date.now();
|
||||
const fileIds = await mlIDbStorage.getFileIds(
|
||||
|
@ -213,19 +213,18 @@ class MachineLearningService {
|
|||
// existingFiles.sort(
|
||||
// (a, b) => b.metadata.creationTime - a.metadata.creationTime
|
||||
// );
|
||||
console.time('getUniqueOutOfSyncFiles');
|
||||
console.time("getUniqueOutOfSyncFiles");
|
||||
syncContext.outOfSyncFiles = await this.getUniqueOutOfSyncFilesNoIdx(
|
||||
syncContext,
|
||||
[...existingFilesMap.values()]
|
||||
[...existingFilesMap.values()],
|
||||
);
|
||||
log.info('getUniqueOutOfSyncFiles');
|
||||
log.info("getUniqueOutOfSyncFiles");
|
||||
log.info(
|
||||
'Got unique outOfSyncFiles: ',
|
||||
"Got unique outOfSyncFiles: ",
|
||||
syncContext.outOfSyncFiles.length,
|
||||
'for batchSize: ',
|
||||
syncContext.config.batchSize
|
||||
"for batchSize: ",
|
||||
syncContext.config.batchSize,
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
private async syncFiles(syncContext: MLSyncContext) {
|
||||
|
@ -441,9 +440,9 @@ class MachineLearningService {
|
|||
}
|
||||
|
||||
private async persistOnServer(mlFileData: MlFileData, enteFile: EnteFile) {
|
||||
const serverMl = LocalFileMlDataToServerFileMl(mlFileData)
|
||||
log.info(mlFileData);
|
||||
|
||||
const serverMl = LocalFileMlDataToServerFileMl(mlFileData);
|
||||
log.info(mlFileData);
|
||||
|
||||
const comlinkCryptoWorker = await ComlinkCryptoWorker.getInstance();
|
||||
const { file: encryptedEmbeddingData } =
|
||||
await comlinkCryptoWorker.encryptMetadata(serverMl, enteFile.key);
|
||||
|
|
|
@ -5,7 +5,7 @@
|
|||
* embeddings on the server. However, we should be prepared to receive an
|
||||
* {@link EncryptedEmbedding} with a model value distinct from one of these.
|
||||
*/
|
||||
export type EmbeddingModel = "onnx-clip" |"file-ml-clip-face";
|
||||
export type EmbeddingModel = "onnx-clip" | "file-ml-clip-face";
|
||||
|
||||
export interface EncryptedEmbedding {
|
||||
fileID: number;
|
||||
|
@ -24,7 +24,6 @@ export interface Embedding
|
|||
embedding?: Float32Array;
|
||||
}
|
||||
|
||||
|
||||
export interface GetEmbeddingDiffResponse {
|
||||
diff: EncryptedEmbedding[];
|
||||
}
|
||||
|
|
|
@ -1,4 +1,9 @@
|
|||
import { Face, FaceDetection, Landmark, MlFileData } from "types/machineLearning";
|
||||
import {
|
||||
Face,
|
||||
FaceDetection,
|
||||
Landmark,
|
||||
MlFileData,
|
||||
} from "types/machineLearning";
|
||||
import { ClipEmbedding } from "types/machineLearning/data/clip";
|
||||
|
||||
export interface FileML extends ServerFileMl {
|
||||
|
@ -12,7 +17,6 @@ class ServerFileMl {
|
|||
public faceEmbedding: ServerFaceEmbeddings;
|
||||
public clipEmbedding?: ClipEmbedding;
|
||||
|
||||
|
||||
public constructor(
|
||||
fileID: number,
|
||||
faceEmbedding: ServerFaceEmbeddings,
|
||||
|
@ -154,12 +158,14 @@ class ServerFaceBox {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
export function LocalFileMlDataToServerFileMl(
|
||||
localFileMlData: MlFileData,
|
||||
): ServerFileMl {
|
||||
if(localFileMlData.errorCount > 0 && localFileMlData.lastErrorMessage !== undefined) {
|
||||
return null
|
||||
if (
|
||||
localFileMlData.errorCount > 0 &&
|
||||
localFileMlData.lastErrorMessage !== undefined
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const imageDimensions = localFileMlData.imageDimensions;
|
||||
const fileInfo = new ServerFileInfo(
|
||||
|
@ -176,22 +182,15 @@ export function LocalFileMlDataToServerFileMl(
|
|||
const detection: FaceDetection = face.detection;
|
||||
const box = detection.box;
|
||||
const landmarks = detection.landmarks;
|
||||
const newBox = new ServerFaceBox(
|
||||
box.x,
|
||||
box.y,
|
||||
box.width,
|
||||
box.height,
|
||||
);
|
||||
const newBox = new ServerFaceBox(box.x, box.y, box.width, box.height);
|
||||
const newLandmarks: Landmark[] = [];
|
||||
for (let j = 0; j < landmarks.length; j++) {
|
||||
newLandmarks.push(
|
||||
{
|
||||
x: landmarks[j].x,
|
||||
y: landmarks[j].y,
|
||||
} as Landmark,
|
||||
);
|
||||
newLandmarks.push({
|
||||
x: landmarks[j].x,
|
||||
y: landmarks[j].y,
|
||||
} as Landmark);
|
||||
}
|
||||
|
||||
|
||||
const newFaceObject = new ServerFace(
|
||||
localFileMlData.fileId,
|
||||
faceID,
|
||||
|
@ -203,7 +202,11 @@ export function LocalFileMlDataToServerFileMl(
|
|||
);
|
||||
faces.push(newFaceObject);
|
||||
}
|
||||
const faceEmbeddings = new ServerFaceEmbeddings(faces, 1,localFileMlData.lastErrorMessage);
|
||||
const faceEmbeddings = new ServerFaceEmbeddings(
|
||||
faces,
|
||||
1,
|
||||
localFileMlData.lastErrorMessage,
|
||||
);
|
||||
return new ServerFileMl(
|
||||
localFileMlData.fileId,
|
||||
faceEmbeddings,
|
||||
|
@ -213,7 +216,6 @@ export function LocalFileMlDataToServerFileMl(
|
|||
);
|
||||
}
|
||||
|
||||
|
||||
// // Not sure if this actually works
|
||||
// export function ServerFileMlToLocalFileMlData(
|
||||
// serverFileMl: ServerFileMl,
|
||||
|
|
Loading…
Add table
Reference in a new issue