12804827031

Committed 16 Jan 2025 08:22AM UTC coverage: 55.394% (-1.2%) from 56.577%

Build # 12804827031

Build Type

push

github

Committed by

nikelborm

Commit Message

quick save: Thu Jan 16 11:22:57 MSK 2025

Coverage Stats

51 of 64 branches covered (79.69%)

Branch coverage included in aggregate %.

178 of 408 new or added lines in 15 files covered. (43.63%)

5 existing lines in 1 file now uncovered.

329 of 622 relevant lines covered (52.89%)

2.32 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.0

/src/getPathContents/parseGitLFSObject.ts

import { isRight, left, mapLeft, match, right } from 'effect/Either';
import { ParseError } from 'effect/ParseResult';
import {
  decodeUnknownEither,
  NonEmptyTrimmedString,
  NumberFromString,
  Struct,
} from 'effect/Schema';
import { TaggedErrorVerifyingCause } from '../TaggedErrorVerifyingCause.js';
import { outdent } from 'outdent';

export const parseGitLFSObject = ({
  contentAsBuffer,
  expectedContentSize,
  blobSha,
  fileName,
  pathToFileInRepo,
}: {
  contentAsBuffer: Buffer<ArrayBuffer>;
  expectedContentSize: number;
  blobSha: string;
  fileName: string;
  pathToFileInRepo: string;
}) => {
  // gitLFS info usually is no longer than MAX_GIT_LFS_INFO_SIZE bytes
  const contentAsString = contentAsBuffer
    .subarray(0, MAX_GIT_LFS_INFO_SIZE)
    .toString('utf8');

  const parsingResult = mapLeft(
    decodeGitLFSInfoSchema(
      contentAsString.match(gitLFSInfoRegexp)?.groups,
    ),
    cause =>
      new FailedToParseGitLFSInfo(cause, {
        partOfContentThatCouldBeGitLFSInfo: contentAsString,
      }),
  );

  const matchedByRegexpAndParsedByEffectSchema = isRight(parsingResult);
  const sizeFromGitLFSInfoAlignsWithExpectedContentSize =
    isRight(parsingResult) &&
    parsingResult.right.size === expectedContentSize;

  const shouldFailIfItIsNotGitLFS =
    contentAsBuffer.byteLength !== expectedContentSize;

  const thisIsGitLFSObject =
    matchedByRegexpAndParsedByEffectSchema &&
    sizeFromGitLFSInfoAlignsWithExpectedContentSize;

  if (thisIsGitLFSObject)
    return right({
      type: 'file',
      name: fileName,
      path: pathToFileInRepo,
      blobSha,
      size: expectedContentSize,
      gitLFSObjectIdSha256: parsingResult.right.oidSha256,
      gitLFSVersion: parsingResult.right.version,
      meta: 'This file can be downloaded as a git-LFS object',
    } as const);

  if (shouldFailIfItIsNotGitLFS)
    return left(
      new InconsistentExpectedAndRealContentSize({
        path: pathToFileInRepo,
        actual: contentAsBuffer.byteLength,
        expected: expectedContentSize,
        gitLFSInfo: parsingResult.pipe(
          match({
            onLeft: left => ({
              meta: 'Failed to parse',
              error: left,
            }),
            onRight: right => ({
              meta: 'Parsed successfully',
              value: right,
            }),
          }),
        ),
      }),
    );

  return right('This is not a git LFS object' as const);
};

// there are some responses that look like
// `version https://git-lfs.github.com/spec/v1
// oid sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
// size 128
// `
// and the only variable thing in it is the size at the end, and I assume
// that supported file size is not greater than 100 GB
const MAX_GIT_LFS_INFO_SIZE = 137;
// Don't add regexp /g modifier, it breaks match groups
const gitLFSInfoRegexp =
  /^version (?<version>https:\/\/git-lfs\.github\.com\/spec\/v1)\noid sha256:(?<oidSha256>[0-9a-f]{64})\nsize (?<size>[1-9][0-9]{0,11})\n$/m;

const GitLFSInfoSchema = Struct({
  version: NonEmptyTrimmedString,
  oidSha256: NonEmptyTrimmedString,
  size: NumberFromString,
});

const decodeGitLFSInfoSchema = decodeUnknownEither(GitLFSInfoSchema, {
  exact: true,
});

export class FailedToParseGitLFSInfo extends TaggedErrorVerifyingCause<{
  partOfContentThatCouldBeGitLFSInfo: string;
}>()(
  'FailedToParseGitLFSInfo',
  `Failed to parse git LFS announcement`,
  ParseError,
) {}

export class InconsistentExpectedAndRealContentSize extends TaggedErrorVerifyingCause<{
  path: string;
  actual: number;
  expected: number;
  gitLFSInfo:
    | {
        meta: 'Parsed successfully';
        value: (typeof GitLFSInfoSchema)['Type'];
      }
    | {
        meta: 'Failed to parse';
        error: FailedToParseGitLFSInfo;
      };
}>()(
  'InconsistentExpectedAndRealContentSize',
  ctx =>
    `Got file ${ctx.path} with size ${ctx.actual} bytes while expecting ${ctx.expected} bytes`,
  void 0,
  {
    comment: outdent({ newline: ' ' })`
    If we weren't successful in parsing it as git LFS object
    announcement using RegExp and Effect.Schema, we just do a basic size
    consistency check. The check implements the second marker of it
    being a Git LFS object as a backup to checking does "content" look
    like a Git LFS object. If GitHub API's "size" field is different
    from actual size of "content" field, it means either our schema with
    regexp fucked up, or GitHub API did. If it doesn't throw, it means
    there's no reason to assume it's a Git LFS object.
  `,
  },
) {}

1	import { isRight, left, mapLeft, match, right } from 'effect/Either';	1✔
2	import { ParseError } from 'effect/ParseResult';
3	import {
4	decodeUnknownEither,
5	NonEmptyTrimmedString,
6	NumberFromString,
7	Struct,
8	} from 'effect/Schema';
9	import { TaggedErrorVerifyingCause } from '../TaggedErrorVerifyingCause.js';
10	import { outdent } from 'outdent';
11
12	export const parseGitLFSObject = ({	1✔
13	contentAsBuffer,	4✔
14	expectedContentSize,	4✔
15	blobSha,	4✔
16	fileName,	4✔
17	pathToFileInRepo,	4✔
18	}: {	4✔
19	contentAsBuffer: Buffer<ArrayBuffer>;
20	expectedContentSize: number;
21	blobSha: string;
22	fileName: string;
23	pathToFileInRepo: string;
24	}) => {	4✔
25	// gitLFS info usually is no longer than MAX_GIT_LFS_INFO_SIZE bytes
26	const contentAsString = contentAsBuffer	4✔
27	.subarray(0, MAX_GIT_LFS_INFO_SIZE)	4✔
28	.toString('utf8');	4✔
29
30	const parsingResult = mapLeft(	4✔
31	decodeGitLFSInfoSchema(	4✔
32	contentAsString.match(gitLFSInfoRegexp)?.groups,	4✔
33	),	4✔
34	cause =>	4✔
35	new FailedToParseGitLFSInfo(cause, {	3✔
36	partOfContentThatCouldBeGitLFSInfo: contentAsString,	3✔
37	}),	3✔
38	);	4✔
39
40	const matchedByRegexpAndParsedByEffectSchema = isRight(parsingResult);	4✔
41	const sizeFromGitLFSInfoAlignsWithExpectedContentSize =	4✔
42	isRight(parsingResult) &&	4✔
43	parsingResult.right.size === expectedContentSize;	1✔
44
45	const shouldFailIfItIsNotGitLFS =	4✔
46	contentAsBuffer.byteLength !== expectedContentSize;	4✔
47
48	const thisIsGitLFSObject =	4✔
49	matchedByRegexpAndParsedByEffectSchema &&	4✔
50	sizeFromGitLFSInfoAlignsWithExpectedContentSize;	1✔
51
52	if (thisIsGitLFSObject)	4✔
53	return right({	4✔
54	type: 'file',	1✔
55	name: fileName,	1✔
56	path: pathToFileInRepo,	1✔
57	blobSha,	1✔
58	size: expectedContentSize,	1✔
59	gitLFSObjectIdSha256: parsingResult.right.oidSha256,	1✔
60	gitLFSVersion: parsingResult.right.version,	1✔
61	meta: 'This file can be downloaded as a git-LFS object',	1✔
62	} as const);	1✔
63
64	if (shouldFailIfItIsNotGitLFS)	3✔
65	return left(	4!
NEW 66	new InconsistentExpectedAndRealContentSize({	×
NEW 67	path: pathToFileInRepo,	×
NEW 68	actual: contentAsBuffer.byteLength,	×
NEW 69	expected: expectedContentSize,	×
NEW 70	gitLFSInfo: parsingResult.pipe(	×
NEW 71	match({	×
NEW 72	onLeft: left => ({	×
NEW 73	meta: 'Failed to parse',	×
NEW 74	error: left,	×
NEW 75	}),	×
NEW 76	onRight: right => ({	×
NEW 77	meta: 'Parsed successfully',	×
NEW 78	value: right,	×
NEW 79	}),	×
NEW 80	}),	×
NEW 81	),	×
NEW 82	}),	×
NEW 83	);	✔
84
85	return right('This is not a git LFS object' as const);	3✔
86	};	3✔
87
88	// there are some responses that look like
89	// `version https://git-lfs.github.com/spec/v1
90	// oid sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
91	// size 128
92	// `
93	// and the only variable thing in it is the size at the end, and I assume
94	// that supported file size is not greater than 100 GB
95	const MAX_GIT_LFS_INFO_SIZE = 137;	1✔
96	// Don't add regexp /g modifier, it breaks match groups
97	const gitLFSInfoRegexp =	1✔
98	/^version (?<version>https:\/\/git-lfs\.github\.com\/spec\/v1)\noid sha256:(?<oidSha256>[0-9a-f]{64})\nsize (?<size>[1-9][0-9]{0,11})\n$/m;	1✔
99
100	const GitLFSInfoSchema = Struct({	1✔
101	version: NonEmptyTrimmedString,	1✔
102	oidSha256: NonEmptyTrimmedString,	1✔
103	size: NumberFromString,	1✔
104	});	1✔
105
106	const decodeGitLFSInfoSchema = decodeUnknownEither(GitLFSInfoSchema, {	1✔
107	exact: true,	1✔
108	});	1✔
109
110	export class FailedToParseGitLFSInfo extends TaggedErrorVerifyingCause<{	1✔
111	partOfContentThatCouldBeGitLFSInfo: string;
112	}>()(	1✔
113	'FailedToParseGitLFSInfo',	1✔
114	`Failed to parse git LFS announcement`,	1✔
115	ParseError,	1✔
116	) {}	1✔
117
118	export class InconsistentExpectedAndRealContentSize extends TaggedErrorVerifyingCause<{	1✔
119	path: string;
120	actual: number;
121	expected: number;
122	gitLFSInfo:
123	\| {
124	meta: 'Parsed successfully';
125	value: (typeof GitLFSInfoSchema)['Type'];
126	}
127	\| {
128	meta: 'Failed to parse';
129	error: FailedToParseGitLFSInfo;
130	};
131	}>()(	1✔
132	'InconsistentExpectedAndRealContentSize',	1✔
133	ctx =>	1✔
134	`Got file ${ctx.path} with size ${ctx.actual} bytes while expecting ${ctx.expected} bytes`,	1✔
135	void 0,	1✔
136	{	1✔
137	comment: outdent({ newline: ' ' })`	1✔
138	If we weren't successful in parsing it as git LFS object
139	announcement using RegExp and Effect.Schema, we just do a basic size
140	consistency check. The check implements the second marker of it
141	being a Git LFS object as a backup to checking does "content" look
142	like a Git LFS object. If GitHub API's "size" field is different
143	from actual size of "content" field, it means either our schema with
144	regexp fucked up, or GitHub API did. If it doesn't throw, it means
145	there's no reason to assume it's a Git LFS object.
146	`,
147	},	1✔
148	) {}	1✔

nikelborm / fetch-github-folder / 12804827031

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous