Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 11 additions & 6 deletions DEVELOPING.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,14 @@ This tests the typescript using ts-node.
### `yarn lint`

This lints all the typescript. If there are no errors/warnings
from tslint, then you get clean output. But, if there are errors from tslint,
you will see a long error that can be confusing – just focus on the tslint
errors. The results of this are deeper than what the tslint extension in VS Code
does because of [semantic lint
rules](https://palantir.github.io/tslint/usage/type-checking/) which requires a
tsconfig.json to be passed to tslint.
from eslint, then you get clean output.

## `yarn test:nuts:local`

real tests on real local fs, including some scale and perf tests.

## `npx knip`

makes sure that you're not introducing unnecessary exports or leaving dead code

unused exports might be used for tests but not by other code
15 changes: 7 additions & 8 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -50,26 +50,25 @@
"node": ">=18.0.0"
},
"dependencies": {
"@oclif/core": "^4.4.0",
"@salesforce/core": "^8.22.0",
"@salesforce/kit": "^3.2.3",
"@salesforce/source-deploy-retrieve": "^12.22.11",
"@salesforce/core": "^8.23.3",
"@salesforce/kit": "^3.2.4",
"@salesforce/source-deploy-retrieve": "^12.24.3",
"@salesforce/ts-types": "^2.0.12",
"fast-xml-parser": "^4.5.3",
"graceful-fs": "^4.2.11",
"isomorphic-git": "^1.30.1",
"isomorphic-git": "^1.34.0",
"ts-retry-promise": "^0.8.1"
},
"devDependencies": {
"@salesforce/cli-plugins-testkit": "^5.3.39",
"@salesforce/dev-scripts": "^11.0.4",
"@salesforce/schemas": "^1.9.1",
"@types/graceful-fs": "^4.1.9",
"esbuild": "^0.25.3",
"eslint-plugin-sf-plugin": "^1.20.26",
"esbuild": "^0.25.10",
"eslint-plugin-sf-plugin": "^1.20.33",
"ts-node": "^10.9.2",
"ts-patch": "^3.3.0",
"typescript": "^5.9.2"
"typescript": "^5.9.3"
},
"config": {},
"publishConfig": {
Expand Down
4 changes: 2 additions & 2 deletions src/shared/functions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ export const ensureRelative =
(filePath: string): string =>
isAbsolute(filePath) ? relative(projectPath, filePath) : filePath;

export type ParsedCustomLabels = {
type ParsedCustomLabels = {
CustomLabels: { labels: Array<{ fullName: string }> };
};

Expand Down Expand Up @@ -180,7 +180,7 @@ export const remoteChangeToMetadataMember = (cr: ChangeResult): MetadataMember =
export const FileResponseSuccessToRemoteSyncInput = (fr: FileResponseSuccess): RemoteSyncInput => fr;

export const changeResultToMetadataComponent =
(registry: RegistryAccess = new RegistryAccess()) =>
(registry: RegistryAccess) =>
(cr: ChangeResultWithNameAndType): MetadataComponent => ({
fullName: cr.name,
type: registry.getTypeByName(cr.type),
Expand Down
16 changes: 1 addition & 15 deletions src/shared/guards.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
MetadataMember,
FileResponse,
ComponentStatus,
FileResponseFailure,
FileResponseSuccess,
} from '@salesforce/source-deploy-retrieve';
import { FileResponse, ComponentStatus, FileResponseSuccess } from '@salesforce/source-deploy-retrieve';
import { ChangeResult } from './types';
import { ChangeResultWithNameAndType } from './types';

export const metadataMemberGuard = (
input: MetadataMember | undefined | Partial<MetadataMember>
): input is MetadataMember =>
input !== undefined && typeof input.fullName === 'string' && typeof input.type === 'string';

export const isSdrFailure = (fileResponse: FileResponse): fileResponse is FileResponseFailure =>
fileResponse.state === ComponentStatus.Failed;

export const isSdrSuccess = (fileResponse: FileResponse): fileResponse is FileResponseSuccess =>
fileResponse.state !== ComponentStatus.Failed;

Expand Down
3 changes: 1 addition & 2 deletions src/shared/local/functions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as os from 'node:os';
import * as path from 'node:path';
import * as os from 'node:os';
import { StatusRow } from './types';
export const IS_WINDOWS = os.type() === 'Windows_NT'; // array members for status results

// filenames were normalized when read from isogit
export const toFilenames = (rows: StatusRow[]): string[] => rows.map((row) => row[FILE]);
export const isDeleted = (status: StatusRow): boolean => status[WORKDIR] === 0;
export const isAdded = (status: StatusRow): boolean => status[HEAD] === 0 && status[WORKDIR] === 2;
export const ensureWindows = (filepath: string): string => path.win32.normalize(filepath);
export const ensurePosix = (filepath: string): string => filepath.split(path.sep).join(path.posix.sep);

// We don't use STAGE (StatusRow[3]). Changes are added and committed in one step
Expand Down
36 changes: 4 additions & 32 deletions src/shared/local/localShadowRepo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@ import * as os from 'node:os';
import * as fs from 'graceful-fs';
import { NamedPackageDir, Lifecycle, Logger, SfError } from '@salesforce/core';
import { env } from '@salesforce/kit';
// @ts-expect-error isogit has both ESM and CJS exports but node16 module/resolution identifies it as ESM
import git from 'isomorphic-git';
import { Performance } from '@oclif/core/performance';
import { RegistryAccess } from '@salesforce/source-deploy-retrieve';
import { chunkArray, excludeLwcLocalOnlyTest, folderContainsPath } from '../functions';
import { filenameMatchesToMap, getLogMessage, getMatches } from './moveDetection';
Expand Down Expand Up @@ -59,8 +57,7 @@ type CommitRequest = {
needsUpdatedStatus?: boolean;
};

/** do not try to add more than this many files at a time through isogit. You'll hit EMFILE: too many open files even with graceful-fs */

/** do not try to add more than this many files at a time through isogit. You'll hit EMFILE: too many open files */
const MAX_FILE_ADD = env.getNumber(
'SF_SOURCE_TRACKING_BATCH_SIZE',
env.getNumber('SFDX_SOURCE_TRACKING_BATCH_SIZE', IS_WINDOWS ? 8000 : 15_000)
Expand Down Expand Up @@ -130,11 +127,7 @@ export class ShadowRepo {
* @returns the deleted directory
*/
public async delete(): Promise<string> {
if (typeof fs.promises.rm === 'function') {
await fs.promises.rm(this.gitDir, { recursive: true, force: true });
} else {
await fs.promises.rm(this.gitDir, { recursive: true });
}
await fs.promises.rm(this.gitDir, { recursive: true, force: true });
return this.gitDir;
}
/**
Expand All @@ -149,8 +142,6 @@ export class ShadowRepo {
this.logger.trace(`start: getStatus (noCache = ${noCache})`);

if (!this.status || noCache) {
const marker = Performance.mark('@salesforce/source-tracking', 'localShadowRepo.getStatus#withoutCache');

try {
// status hasn't been initialized yet
this.status = await git.statusMatrix({
Expand All @@ -177,8 +168,6 @@ export class ShadowRepo {
} catch (e) {
redirectToCliRepoError(e);
}

marker?.stop();
}
this.logger.trace(`done: getStatus (noCache = ${noCache})`);
return this.status;
Expand Down Expand Up @@ -259,11 +248,6 @@ export class ShadowRepo {
return 'no files to commit';
}

const marker = Performance.mark('@salesforce/source-tracking', 'localShadowRepo.commitChanges', {
deployedFiles: deployedFiles.length,
deletedFiles: deletedFiles.length,
});

if (deployedFiles.length) {
const chunks = chunkArray(
// these are stored in posix/style/path format. We have to convert inbound stuff from windows
Expand Down Expand Up @@ -303,9 +287,7 @@ export class ShadowRepo {
if (deletedFiles.length) {
// Using a cache here speeds up the performance by ~24.4%
let cache = {};
const deleteMarker = Performance.mark('@salesforce/source-tracking', 'localShadowRepo.commitChanges#delete', {
deletedFiles: deletedFiles.length,
});

for (const filepath of [...new Set(IS_WINDOWS ? deletedFiles.map(normalize).map(ensurePosix) : deletedFiles)]) {
try {
// these need to be done sequentially because isogit manages file locking. Isogit remove does not support multiple files at once
Expand All @@ -317,7 +299,6 @@ export class ShadowRepo {
}
// clear cache
cache = {};
deleteMarker?.stop();
}

try {
Expand All @@ -339,34 +320,25 @@ export class ShadowRepo {
} catch (e) {
redirectToCliRepoError(e);
}
marker?.stop();
}

private async detectMovedFiles(): Promise<void> {
// get status will return os-specific paths
const matchingFiles = getMatches(await this.getStatus());
if (!matchingFiles.added.size || !matchingFiles.deleted.size) return;

const movedFilesMarker = Performance.mark('@salesforce/source-tracking', 'localShadowRepo.detectMovedFiles');
const matches = await filenameMatchesToMap(this.registry)(this.projectPath)(this.gitDir)(matchingFiles);

if (matches.deleteOnly.size === 0 && matches.fullMatches.size === 0) return movedFilesMarker?.stop();
if (matches.deleteOnly.size === 0 && matches.fullMatches.size === 0) return;

this.logger.debug(getLogMessage(matches));

movedFilesMarker?.addDetails({
filesMoved: matches.fullMatches.size,
filesMovedAndEdited: matches.deleteOnly.size,
});

// Commit the moved files and refresh the status
await this.commitChanges({
deletedFiles: [...matches.fullMatches.values(), ...matches.deleteOnly.values()],
deployedFiles: [...matches.fullMatches.keys()],
message: 'Committing moved files',
});

movedFilesMarker?.stop();
}
}

Expand Down
14 changes: 3 additions & 11 deletions src/shared/local/moveDetection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,15 @@
*/
import path from 'node:path';
import { EOL } from 'node:os';
import * as fs from 'graceful-fs';
import { Logger, Lifecycle } from '@salesforce/core';
import {
MetadataResolver,
SourceComponent,
RegistryAccess,
VirtualTreeContainer,
} from '@salesforce/source-deploy-retrieve';
// @ts-expect-error isogit has both ESM and CJS exports but node16 module/resolution identifies it as ESM
import git from 'isomorphic-git';
import * as fs from 'graceful-fs';
import { Performance } from '@oclif/core/performance';
import { isDefined } from '../guards';
import { uniqueArrayConcat } from '../functions';
import { isDeleted, isAdded, toFilenames, IS_WINDOWS, ensurePosix } from './functions';
Expand Down Expand Up @@ -178,24 +176,18 @@ const toFileInfo = async ({
deleted: Set<string>;
}): Promise<AddAndDeleteFileInfos> => {
// Track how long it takes to gather the oid information from the git trees
const getInfoMarker = Performance.mark('@salesforce/source-tracking', 'localShadowRepo.detectMovedFiles#toFileInfo', {
addedFiles: added.size,
deletedFiles: deleted.size,
});

const headRef = await git.resolveRef({ fs, dir: projectPath, gitdir: gitDir, ref: 'HEAD' });
const [addedInfo, deletedInfo] = await Promise.all([
await Promise.all(Array.from(added).map(getHashForAddedFile(projectPath))),
await Promise.all(Array.from(deleted).map(getHashFromActualFileContents(gitDir)(projectPath)(headRef))),
]);

getInfoMarker?.stop();

return { addedInfo, deletedInfo };
};

/** returns a map of <hash+basename, filepath>. If two items result in the same hash+basename, return that in the ignore bucket */
export const buildMap = (info: DetectionFileInfoWithType[]): StringMap[] => {
const buildMap = (info: DetectionFileInfoWithType[]): StringMap[] => {
const map: StringMap = new Map();
const ignore: StringMap = new Map();

Expand Down Expand Up @@ -254,7 +246,7 @@ const getHashFromActualFileContents =
).oid,
});

export const toKey = (input: DetectionFileInfoWithType): string =>
const toKey = (input: DetectionFileInfoWithType): string =>
[input.hash, input.basename, input.type, input.type, input.parentType ?? '', input.parentFullName ?? ''].join(
JOIN_CHAR
);
Expand Down
2 changes: 1 addition & 1 deletion src/shared/localComponentSetArray.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ const getNonSequential = ({
export const getComponentSets = ({
groupings,
sourceApiVersion,
registry = new RegistryAccess(),
registry,
}: {
groupings: GroupedFile[];
sourceApiVersion?: string;
Expand Down
Loading
Loading