12804526304

Committed 16 Jan 2025 08:03AM UTC coverage: 56.577% (+0.2%) from 56.366%

Build # 12804526304

Build Type

push

github

Committed by

nikelborm

Commit Message

quick save: Thu Jan 16 11:03:21 MSK 2025

Coverage Stats

47 of 59 branches covered (79.66%)

Branch coverage included in aggregate %.

0 of 1 new or added line in 1 file covered. (0.0%)

75 existing lines in 8 files now uncovered.

310 of 572 relevant lines covered (54.2%)

2.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.52

/src/getPathContents/parseGitLFSObject.ts

import { isRight, left, mapLeft, match, right } from 'effect/Either';
import { ParseError } from 'effect/ParseResult';
import { decodeUnknownEither, NonEmptyTrimmedString, NumberFromString, Struct } from 'effect/Schema';
import { TaggedErrorVerifyingCause } from '../TaggedErrorVerifyingCause.js';
import { outdent } from 'outdent';

export const parseGitLFSObject = ({
  contentAsBuffer,
  expectedContentSize,
  blobSha,
  fileName,
  pathToFileInRepo,
}: {
  contentAsBuffer: Buffer<ArrayBuffer>,
  expectedContentSize: number,
  blobSha: string,
  fileName: string;
  pathToFileInRepo: string;
}) => {
  // gitLFS info usually is no longer than MAX_GIT_LFS_INFO_SIZE bytes
  const contentAsString = contentAsBuffer
    .subarray(0, MAX_GIT_LFS_INFO_SIZE)
    .toString("utf8");

  const parsingResult = mapLeft(
    decodeGitLFSInfoSchema(
      contentAsString.match(gitLFSInfoRegexp)?.groups
    ),
    cause => new FailedToParseGitLFSInfo(
      cause,
      { partOfContentThatCouldBeGitLFSInfo: contentAsString }
    )
  );

  const matchedByRegexpAndParsedByEffectSchema = isRight(parsingResult);
  const sizeFromGitLFSInfoAlignsWithExpectedContentSize = (
    isRight(parsingResult)
    && (parsingResult.right.size === expectedContentSize)
  );

  const shouldFailIfItIsNotGitLFS = contentAsBuffer.byteLength !== expectedContentSize;

  const thisIsGitLFSObject = matchedByRegexpAndParsedByEffectSchema
    && sizeFromGitLFSInfoAlignsWithExpectedContentSize;

  if (thisIsGitLFSObject) return right({
    type: 'file',
    name: fileName,
    path: pathToFileInRepo,
    blobSha,
    size: expectedContentSize,
    gitLFSObjectIdSha256: parsingResult.right.oidSha256,
    gitLFSVersion: parsingResult.right.version,
    meta: "This file can be downloaded as a git-LFS object"
  } as const);

  if (shouldFailIfItIsNotGitLFS) return left(
    new InconsistentExpectedAndRealContentSize({
      path: pathToFileInRepo,
      actual: contentAsBuffer.byteLength,
      expected: expectedContentSize,
      gitLFSInfo: parsingResult.pipe(match({
        onLeft: left => ({
          meta: 'Failed to parse',
          error: left
        }),
        onRight: right => ({
          meta: 'Parsed successfully',
          value: right
        }),
      }))
    })
  );

  return right("This is not a git LFS object" as const);
}


// there are some responses that look like
// `version https://git-lfs.github.com/spec/v1
// oid sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
// size 128
// `
// and the only variable thing in it is the size at the end, and I assume
// that supported file size is not greater than 100 GB
const MAX_GIT_LFS_INFO_SIZE = 137;
// Don't add regexp /g modifier, it breaks match groups
const gitLFSInfoRegexp = /^version (?<version>https:\/\/git-lfs\.github\.com\/spec\/v1)\noid sha256:(?<oidSha256>[0-9a-f]{64})\nsize (?<size>[1-9][0-9]{0,11})\n$/m

const GitLFSInfoSchema = Struct({
  version: NonEmptyTrimmedString,
  oidSha256: NonEmptyTrimmedString,
  size: NumberFromString
})


const decodeGitLFSInfoSchema = decodeUnknownEither(
  GitLFSInfoSchema,
  { exact: true }
);


export class FailedToParseGitLFSInfo extends TaggedErrorVerifyingCause<{
  partOfContentThatCouldBeGitLFSInfo: string,
}>()(
  'FailedToParseGitLFSInfo',
  `Failed to parse git LFS announcement`,
  ParseError
) {}

export class InconsistentExpectedAndRealContentSize extends TaggedErrorVerifyingCause<{
  path: string,
  actual: number,
  expected: number,
  gitLFSInfo: {
    meta: 'Parsed successfully'
    value: (typeof GitLFSInfoSchema)['Type']
  } | {
    meta: 'Failed to parse'
    error: FailedToParseGitLFSInfo
  }
}>()(
  'InconsistentExpectedAndRealContentSize',
  (ctx) => `Got file ${ctx.path} with size ${ctx.actual} bytes while expecting ${ctx.expected} bytes`,
  void 0,
  { comment: outdent({ newline: ' ' })`
    If we weren't successful in parsing it as git LFS object
    announcement using RegExp and Effect.Schema, we just do a basic size
    consistency check. The check implements the second marker of it
    being a Git LFS object as a backup to checking does "content" look
    like a Git LFS object. If GitHub API's "size" field is different
    from actual size of "content" field, it means either our schema with
    regexp fucked up, or GitHub API did. If it doesn't throw, it means
    there's no reason to assume it's a Git LFS object.
  ` }
) {}

1	import { isRight, left, mapLeft, match, right } from 'effect/Either';	1✔
2	import { ParseError } from 'effect/ParseResult';
3	import { decodeUnknownEither, NonEmptyTrimmedString, NumberFromString, Struct } from 'effect/Schema';
4	import { TaggedErrorVerifyingCause } from '../TaggedErrorVerifyingCause.js';
5	import { outdent } from 'outdent';
6
7	export const parseGitLFSObject = ({	1✔
8	contentAsBuffer,	4✔
9	expectedContentSize,	4✔
10	blobSha,	4✔
11	fileName,	4✔
12	pathToFileInRepo,	4✔
13	}: {	4✔
14	contentAsBuffer: Buffer<ArrayBuffer>,
15	expectedContentSize: number,
16	blobSha: string,
17	fileName: string;
18	pathToFileInRepo: string;
19	}) => {	4✔
20	// gitLFS info usually is no longer than MAX_GIT_LFS_INFO_SIZE bytes
21	const contentAsString = contentAsBuffer	4✔
22	.subarray(0, MAX_GIT_LFS_INFO_SIZE)	4✔
23	.toString("utf8");	4✔
24
25	const parsingResult = mapLeft(	4✔
26	decodeGitLFSInfoSchema(	4✔
27	contentAsString.match(gitLFSInfoRegexp)?.groups	4✔
28	),	4✔
29	cause => new FailedToParseGitLFSInfo(	4✔
30	cause,	3✔
31	{ partOfContentThatCouldBeGitLFSInfo: contentAsString }	3✔
32	)	3✔
33	);	4✔
34
35	const matchedByRegexpAndParsedByEffectSchema = isRight(parsingResult);	4✔
36	const sizeFromGitLFSInfoAlignsWithExpectedContentSize = (	4✔
37	isRight(parsingResult)	4✔
38	&& (parsingResult.right.size === expectedContentSize)	1✔
39	);
40
41	const shouldFailIfItIsNotGitLFS = contentAsBuffer.byteLength !== expectedContentSize;	4✔
42
43	const thisIsGitLFSObject = matchedByRegexpAndParsedByEffectSchema	4✔
44	&& sizeFromGitLFSInfoAlignsWithExpectedContentSize;	1✔
45
46	if (thisIsGitLFSObject) return right({	4✔
47	type: 'file',	1✔
48	name: fileName,	1✔
49	path: pathToFileInRepo,	1✔
50	blobSha,	1✔
51	size: expectedContentSize,	1✔
52	gitLFSObjectIdSha256: parsingResult.right.oidSha256,	1✔
53	gitLFSVersion: parsingResult.right.version,	1✔
54	meta: "This file can be downloaded as a git-LFS object"	1✔
55	} as const);	1✔
56
57	if (shouldFailIfItIsNotGitLFS) return left(	4!
58	new InconsistentExpectedAndRealContentSize({	×
59	path: pathToFileInRepo,	×
60	actual: contentAsBuffer.byteLength,	×
61	expected: expectedContentSize,	×
62	gitLFSInfo: parsingResult.pipe(match({	×
63	onLeft: left => ({	×
64	meta: 'Failed to parse',	×
65	error: left	×
66	}),	×
67	onRight: right => ({	×
68	meta: 'Parsed successfully',	×
69	value: right	×
70	}),	×
71	}))	×
72	})	×
UNCOV 73	);	✔
74
75	return right("This is not a git LFS object" as const);	3✔
76	}	3✔
77
78
79	// there are some responses that look like
80	// `version https://git-lfs.github.com/spec/v1
81	// oid sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
82	// size 128
83	// `
84	// and the only variable thing in it is the size at the end, and I assume
85	// that supported file size is not greater than 100 GB
86	const MAX_GIT_LFS_INFO_SIZE = 137;	1✔
87	// Don't add regexp /g modifier, it breaks match groups
88	const gitLFSInfoRegexp = /^version (?<version>https:\/\/git-lfs\.github\.com\/spec\/v1)\noid sha256:(?<oidSha256>[0-9a-f]{64})\nsize (?<size>[1-9][0-9]{0,11})\n$/m	1✔
89
90	const GitLFSInfoSchema = Struct({	1✔
91	version: NonEmptyTrimmedString,	1✔
92	oidSha256: NonEmptyTrimmedString,	1✔
93	size: NumberFromString	1✔
94	})	1✔
95
96
97	const decodeGitLFSInfoSchema = decodeUnknownEither(	1✔
98	GitLFSInfoSchema,	1✔
99	{ exact: true }	1✔
100	);	1✔
101
102
103	export class FailedToParseGitLFSInfo extends TaggedErrorVerifyingCause<{	1✔
104	partOfContentThatCouldBeGitLFSInfo: string,
105	}>()(	1✔
106	'FailedToParseGitLFSInfo',	1✔
107	`Failed to parse git LFS announcement`,	1✔
108	ParseError	1✔
109	) {}	1✔
110
111	export class InconsistentExpectedAndRealContentSize extends TaggedErrorVerifyingCause<{	1✔
112	path: string,
113	actual: number,
114	expected: number,
115	gitLFSInfo: {
116	meta: 'Parsed successfully'
117	value: (typeof GitLFSInfoSchema)['Type']
118	} \| {
119	meta: 'Failed to parse'
120	error: FailedToParseGitLFSInfo
121	}
122	}>()(	1✔
123	'InconsistentExpectedAndRealContentSize',	1✔
124	(ctx) => `Got file ${ctx.path} with size ${ctx.actual} bytes while expecting ${ctx.expected} bytes`,	1✔
125	void 0,	1✔
126	{ comment: outdent({ newline: ' ' })`	1✔
127	If we weren't successful in parsing it as git LFS object
128	announcement using RegExp and Effect.Schema, we just do a basic size
129	consistency check. The check implements the second marker of it
130	being a Git LFS object as a backup to checking does "content" look
131	like a Git LFS object. If GitHub API's "size" field is different
132	from actual size of "content" field, it means either our schema with
133	regexp fucked up, or GitHub API did. If it doesn't throw, it means
134	there's no reason to assume it's a Git LFS object.
135	` }	1✔
136	) {}	1✔

nikelborm / fetch-github-folder / 12804526304

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous