Skip to content

Commit

Permalink
File match include exclude
Browse files Browse the repository at this point in the history
  • Loading branch information
@angelini
angelini committed Aug 9, 2023
1 parent 1dead06 commit 48c9c34
Show file tree
Hide file tree
Showing 7 changed files with 226 additions and 112 deletions.
65 changes: 40 additions & 25 deletions internal/files/writer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,47 @@ import (
"github.com/gobwas/glob"
)

type FilePattern struct {
Iff bool
pattern glob.Glob
type FileMatcher struct {
include *glob.Glob
exclude *glob.Glob
}

func NewFilePattern(pattern string, iff bool) (*FilePattern, error) {
glob, err := glob.Compile(pattern)
if err != nil {
return nil, err
func NewFileMatcher(include, exclude string) (*FileMatcher, error) {
matcher := FileMatcher{}

if include != "" {
includeGlob, err := glob.Compile(include)
if err != nil {
return nil, fmt.Errorf("error parsing include file match: %w", err)
}
matcher.include = &includeGlob
}

if exclude != "" {
excludeGlob, err := glob.Compile(exclude)
if err != nil {
return nil, fmt.Errorf("error parsing exclude file match: %w", err)
}
matcher.exclude = &excludeGlob
}

return &FilePattern{
Iff: iff,
pattern: glob,
}, nil
return &matcher, nil
}

func (f *FilePattern) Match(filename string) bool {
return f.pattern.Match(filename)
func (f *FileMatcher) Match(filename string) bool {
result := false

if f.include != nil {
result = (*f.include).Match(filename)
} else {
result = true
}

if result && f.exclude != nil {
result = !(*f.exclude).Match(filename)
}

return result
}

func fileExists(path string) bool {
Expand Down Expand Up @@ -203,12 +225,11 @@ func hardlinkDir(olddir, newdir string) error {
})
}

func WriteTar(finalDir string, cacheObjectsDir string, reader *db.TarReader, packPath *string, pattern *FilePattern) (uint32, bool, error) {
func WriteTar(finalDir string, cacheObjectsDir string, reader *db.TarReader, packPath *string, matcher *FileMatcher) (uint32, bool, error) {
var count uint32
dir := finalDir

patternMatch := false
patternExclusiveMatch := true
fileMatch := true

existingDirs := make(map[string]bool)

Expand All @@ -230,14 +251,8 @@ func WriteTar(finalDir string, cacheObjectsDir string, reader *db.TarReader, pac
return count, false, fmt.Errorf("read next TAR header: %w", err)
}

if pattern != nil {
if !patternMatch && pattern.Match(header.Name) {
patternMatch = true
}

if pattern.Iff && patternExclusiveMatch && !pattern.Match(header.Name) {
patternExclusiveMatch = false
}
if matcher != nil && !matcher.Match(header.Name) {
fileMatch = false
}

err = writeObject(dir, cacheObjectsDir, reader, header, existingDirs)
Expand All @@ -263,5 +278,5 @@ func WriteTar(finalDir string, cacheObjectsDir string, reader *db.TarReader, pac
}
}

return count, patternMatch && patternExclusiveMatch, nil
return count, fileMatch, nil
}
70 changes: 64 additions & 6 deletions js/spec/binary-client.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ describe("binary client operations", () => {
expect(fileContent).toBe(content);
});

it("can rebuild the file system with a file pattern", async () => {
it("can rebuild the file system with a file matcher", async () => {
const project = 1337n;
const path = "hello.txt";
const content = "hello world";
Expand All @@ -64,9 +64,9 @@ describe("binary client operations", () => {
});

const dir = tmpdir();
const result = await binaryClient.rebuild(project, null, dir, { filePattern: "hello*" });
const result = await binaryClient.rebuild(project, null, dir, { matchInclude: "hello*" });
expect(result?.version).toBe(1n);
expect(result?.patternMatch).toBe(true);
expect(result?.fileMatch).toBe(true);

const filepath = `${dir}/${path}`;
expect(fs.existsSync(filepath)).toBe(true);
Expand All @@ -75,7 +75,7 @@ describe("binary client operations", () => {
expect(fileContent).toBe(content);
});

it("can rebuild the file system with an iff file pattern", async () => {
it("can rebuild the file system with a file matcher exclude pattern", async () => {
const project = 1337n;
const path = "hello.txt";
const content = "hello world";
Expand All @@ -100,9 +100,9 @@ describe("binary client operations", () => {
});

const dir = tmpdir();
const result = await binaryClient.rebuild(project, null, dir, { filePattern: "hello*", filePatternIff: true });
const result = await binaryClient.rebuild(project, null, dir, { matchExclude: "bar*" });
expect(result?.version).toBe(2n);
expect(result?.patternMatch).toBe(false);
expect(result?.fileMatch).toBe(false);

const filepath = `${dir}/bar.txt`;
expect(fs.existsSync(filepath)).toBe(true);
Expand Down Expand Up @@ -132,3 +132,61 @@ describe("binary client operations", () => {
expect(result.count).toStrictEqual(0);
});
});

describe("Gadget file match tests", () => {
const writeFilesWithMatcher = async (project: bigint, include: string, exclude: string, paths: string[]): Promise<boolean> => {
const content = "example";
const encodedContent = encodeContent(content);

await grpcClient.newProject(project, []);

for (const path of paths) {
await grpcClient.updateObject(project, {
path,
mode: 0o755n,
content: encodedContent,
size: BigInt(encodedContent.length),
deleted: false,
});
}

const dir = tmpdir();
const result = await binaryClient.rebuild(project, null, dir, { matchInclude: include, matchExclude: exclude });

fs.rmSync(dir, { recursive: true });

return result.fileMatch;
};

it("can support Gadget's pattern", async () => {
let result = await writeFilesWithMatcher(5n, "frontend/**", "frontend/vite.config.[jt]s", [
"frontend/example.js",
"frontend/other.css",
"frontend/third.jsx",
]);
expect(result).toBe(true);

result = await writeFilesWithMatcher(6n, "frontend/**", "frontend/vite.config.[jt]s", [
"frontend/example.js",
"frontend/other.css",
"frontend/third.jsx",
"frontend/vite.config.js",
]);
expect(result).toBe(false);

result = await writeFilesWithMatcher(7n, "frontend/**", "frontend/vite.config.[jt]s", [
"frontend/example.js",
"frontend/other.css",
"frontend/third.jsx",
"frontend/vite.config.ts",
]);
expect(result).toBe(false);

result = await writeFilesWithMatcher(8n, "frontend/**", "frontend/vite.config.[jt]s", [
"frontend/example.js",
"frontend/other.css",
"model/effect.js",
]);
expect(result).toBe(false);
});
});
44 changes: 22 additions & 22 deletions js/src/binary-client.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,9 +106,9 @@ export interface RebuildResult {
*/
count: number;
/**
* Wether or not the file pattern was detected.
* Whether or not the file match was detected.
*/
patternMatch: boolean;
fileMatch: boolean;
}

/**
Expand Down Expand Up @@ -188,17 +188,17 @@ export class DateiLagerBinaryClient {
/**
* Rebuild the local filesystem.
*
* @param project The id of the project.
* @param to The version of the project to rebuild the filesystem to.
* @param directory The path of the directory to rebuild the filesystem at.
* @param options Object of options.
* @param options.timeout Number of milliseconds to wait before terminating the process.
* @param options.ignores The paths to ignore when rebuilding the FS.
* @param options.summarize Should produce the summary file after rebuilding.
* @param options.cacheDir Path where the cache directory is mounted.
* @param options.filePattern A glob file pattern which drives the patternDetected output boolean
* @param options.filePatternIff Should the file pattern detection trigger if and only if those files have changed
* @returns The latest project version or `null` if something went wrong.
* @param project The id of the project.
* @param to The version of the project to rebuild the filesystem to.
* @param directory The path of the directory to rebuild the filesystem at.
* @param options Object of options.
* @param options.timeout Number of milliseconds to wait before terminating the process.
* @param options.ignores The paths to ignore when rebuilding the FS.
* @param options.summarize Should produce the summary file after rebuilding.
* @param options.cacheDir Path where the cache directory is mounted.
* @param options.matchInclude Set fileMatch to true if the written files are matched by this glob pattern
* @param options.matchExclude Set fileMatch to false if the written files are matched by this glob pattern
* @returns The latest project version or `null` if something went wrong.
*/
public async rebuild(
project: bigint,
Expand All @@ -209,8 +209,8 @@ export class DateiLagerBinaryClient {
ignores?: string[];
summarize?: boolean;
cacheDir?: string;
filePattern?: string;
filePatternIff?: boolean;
matchInclude?: string;
matchExclude?: string;
}
): Promise<RebuildResult> {
return await trace(
Expand Down Expand Up @@ -242,18 +242,18 @@ export class DateiLagerBinaryClient {
args.push(`--cachedir=${options.cacheDir}`);
}

if (options?.filePattern) {
args.push(`--filepattern=${options.filePattern}`);
if (options?.matchInclude) {
args.push(`--matchinclude=${options.matchInclude}`);
}

if (options.filePatternIff) {
args.push(`--iff=true`);
}
if (options?.matchExclude) {
args.push(`--matchexclude=${options.matchExclude}`);
}

args.push("--project", String(project), "--dir", directory);
const result = await this._call("rebuild", args, directory, options);
const parsed = JSON.parse(result.stdout) as { version: number; count: number; patternMatch: boolean };
return { version: BigInt(parsed.version), count: parsed.count, patternMatch: parsed.patternMatch };
const parsed = JSON.parse(result.stdout) as { version: number; count: number; fileMatch: boolean };
return { version: BigInt(parsed.version), count: parsed.count, fileMatch: parsed.fileMatch };
}
);
}
Expand Down
34 changes: 15 additions & 19 deletions pkg/cli/rebuild.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,15 @@ import (

func NewCmdRebuild() *cobra.Command {
var (
project int64
to *int64
prefix string
dir string
ignores string
summarize bool
cacheDir string
filePattern string
filePatternIff bool
project int64
to *int64
prefix string
dir string
ignores string
summarize bool
cacheDir string
fileMatchInclude string
fileMatchExclude string
)

cmd := &cobra.Command{
Expand All @@ -40,16 +40,12 @@ func NewCmdRebuild() *cobra.Command {
ignoreList = strings.Split(ignores, ",")
}

var pattern *files.FilePattern
if filePattern != "" {
var err error
pattern, err = files.NewFilePattern(filePattern, filePatternIff)
if err != nil {
return fmt.Errorf("invalid file pattern: %w", err)
}
matcher, err := files.NewFileMatcher(fileMatchInclude, fileMatchExclude)
if err != nil {
return err
}

result, err := client.Rebuild(ctx, project, prefix, to, dir, ignoreList, cacheDir, pattern, summarize)
result, err := client.Rebuild(ctx, project, prefix, to, dir, ignoreList, cacheDir, matcher, summarize)
if err != nil {
return fmt.Errorf("could not rebuild project: %w", err)
}
Expand Down Expand Up @@ -79,8 +75,8 @@ func NewCmdRebuild() *cobra.Command {
cmd.Flags().StringVar(&ignores, "ignores", "", "Comma separated list of ignore paths")
cmd.Flags().BoolVar(&summarize, "summarize", true, "Should include the summary file (required for future updates)")
cmd.Flags().StringVar(&cacheDir, "cachedir", "", "Path where the cache folder is mounted")
cmd.Flags().StringVar(&filePattern, "filepattern", "", "A glob file pattern which drives the patternMatch output boolean")
cmd.Flags().BoolVar(&filePatternIff, "iff", false, "Should the file pattern detection trigger if and only if those files have changed")
cmd.Flags().StringVar(&fileMatchInclude, "matchinclude", "", "Set fileMatch to true if the written files are matched by this glob pattern")
cmd.Flags().StringVar(&fileMatchExclude, "matchexclude", "", "Set fileMatch to false if the written files are matched by this glob pattern")
to = cmd.Flags().Int64("to", -1, "To version ID (optional)")

_ = cmd.MarkFlagRequired("project")
Expand Down
Loading

0 comments on commit 48c9c34

Please sign in to comment.