diff --git a/eszip-go/checksum.go b/eszip-go/checksum.go new file mode 100644 index 0000000..ff67b64 --- /dev/null +++ b/eszip-go/checksum.go @@ -0,0 +1,82 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +package eszip + +import ( + "bytes" + "crypto/sha256" + + "github.com/zeebo/xxh3" +) + +// ChecksumType represents the hash algorithm used for checksums +type ChecksumType uint8 + +const ( + ChecksumNone ChecksumType = 0 + ChecksumSha256 ChecksumType = 1 + ChecksumXxh3 ChecksumType = 2 +) + +// DigestSize returns the size in bytes of the hash digest +func (c ChecksumType) DigestSize() uint8 { + switch c { + case ChecksumNone: + return 0 + case ChecksumSha256: + return 32 + case ChecksumXxh3: + return 8 + default: + return 0 + } +} + +// Hash computes the checksum of the given data +func (c ChecksumType) Hash(data []byte) []byte { + switch c { + case ChecksumNone: + return nil + case ChecksumSha256: + h := sha256.Sum256(data) + return h[:] + case ChecksumXxh3: + h := xxh3.Hash(data) + // Convert to big-endian bytes + return []byte{ + byte(h >> 56), + byte(h >> 48), + byte(h >> 40), + byte(h >> 32), + byte(h >> 24), + byte(h >> 16), + byte(h >> 8), + byte(h), + } + default: + return nil + } +} + +// Verify checks if the given hash matches the data +func (c ChecksumType) Verify(data, hash []byte) bool { + if c == ChecksumNone { + return true + } + computed := c.Hash(data) + return bytes.Equal(computed, hash) +} + +// FromU8 creates a ChecksumType from a byte value +func ChecksumFromU8(b uint8) (ChecksumType, bool) { + switch b { + case 0: + return ChecksumNone, true + case 1: + return ChecksumSha256, true + case 2: + return ChecksumXxh3, true + default: + return ChecksumNone, false + } +} diff --git a/eszip-go/cmd/eszip/main.go b/eszip-go/cmd/eszip/main.go new file mode 100644 index 0000000..b211f34 --- /dev/null +++ b/eszip-go/cmd/eszip/main.go @@ -0,0 +1,402 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +// eszip is a CLI tool for working with eszip archives. +package main + +import ( + "context" + "flag" + "fmt" + "os" + "path/filepath" + "strings" + + eszip "github.com/example/eszip-go" +) + +func main() { + if len(os.Args) < 2 { + printUsage() + os.Exit(1) + } + + command := os.Args[1] + + switch command { + case "view", "v": + viewCmd(os.Args[2:]) + case "extract", "x": + extractCmd(os.Args[2:]) + case "create", "c": + createCmd(os.Args[2:]) + case "info", "i": + infoCmd(os.Args[2:]) + case "help", "-h", "--help": + printUsage() + default: + fmt.Fprintf(os.Stderr, "Unknown command: %s\n\n", command) + printUsage() + os.Exit(1) + } +} + +func printUsage() { + fmt.Println(`eszip - A tool for working with eszip archives + +Usage: + eszip [options] + +Commands: + view, v View contents of an eszip archive + extract, x Extract files from an eszip archive + create, c Create a new eszip archive from files + info, i Show information about an eszip archive + help Show this help message + +Examples: + eszip view archive.eszip2 + eszip view -s file:///main.ts archive.eszip2 + eszip extract -o ./output archive.eszip2 + eszip create -o archive.eszip2 file1.js file2.js + eszip info archive.eszip2 + +Run 'eszip -h' for more information on a command.`) +} + +// viewCmd handles the 'view' command +func viewCmd(args []string) { + fs := flag.NewFlagSet("view", flag.ExitOnError) + specifier := fs.String("s", "", "Show only this specifier") + showSourceMap := fs.Bool("m", false, "Show source maps") + fs.Usage = func() { + fmt.Println(`Usage: eszip view [options] + +View the contents of an eszip archive. + +Options:`) + fs.PrintDefaults() + } + + fs.Parse(args) + if fs.NArg() < 1 { + fs.Usage() + os.Exit(1) + } + + archivePath := fs.Arg(0) + ctx := context.Background() + + archive, err := loadArchive(ctx, archivePath) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + + specifiers := archive.Specifiers() + for _, spec := range specifiers { + if *specifier != "" && spec != *specifier { + continue + } + + module := archive.GetModule(spec) + if module == nil { + // Might be a redirect-only or npm specifier + continue + } + + fmt.Printf("Specifier: %s\n", spec) + fmt.Printf("Kind: %s\n", module.Kind) + fmt.Println("---") + + source, err := module.Source(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting source: %v\n", err) + continue + } + + if source != nil { + fmt.Println(string(source)) + } else { + fmt.Println("(source taken)") + } + + if *showSourceMap { + sourceMap, err := module.SourceMap(ctx) + if err == nil && len(sourceMap) > 0 { + fmt.Println("--- Source Map ---") + fmt.Println(string(sourceMap)) + } + } + + fmt.Println("============") + } +} + +// extractCmd handles the 'extract' command +func extractCmd(args []string) { + fs := flag.NewFlagSet("extract", flag.ExitOnError) + outputDir := fs.String("o", ".", "Output directory") + fs.Usage = func() { + fmt.Println(`Usage: eszip extract [options] + +Extract files from an eszip archive. + +Options:`) + fs.PrintDefaults() + } + + fs.Parse(args) + if fs.NArg() < 1 { + fs.Usage() + os.Exit(1) + } + + archivePath := fs.Arg(0) + ctx := context.Background() + + archive, err := loadArchive(ctx, archivePath) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + + specifiers := archive.Specifiers() + for _, spec := range specifiers { + module := archive.GetModule(spec) + if module == nil { + continue + } + + // Skip data: URLs + if strings.HasPrefix(spec, "data:") { + continue + } + + source, err := module.Source(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting source for %s: %v\n", spec, err) + continue + } + + if source == nil { + continue + } + + // Convert specifier to file path + filePath := specifierToPath(spec) + fullPath := filepath.Join(*outputDir, filePath) + + // Create parent directories + if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { + fmt.Fprintf(os.Stderr, "Error creating directory: %v\n", err) + continue + } + + // Write file + if err := os.WriteFile(fullPath, source, 0644); err != nil { + fmt.Fprintf(os.Stderr, "Error writing file: %v\n", err) + continue + } + + fmt.Printf("Extracted: %s\n", fullPath) + + // Also extract source map if available + sourceMap, err := module.SourceMap(ctx) + if err == nil && len(sourceMap) > 0 { + mapPath := fullPath + ".map" + if err := os.WriteFile(mapPath, sourceMap, 0644); err == nil { + fmt.Printf("Extracted: %s\n", mapPath) + } + } + } +} + +// createCmd handles the 'create' command +func createCmd(args []string) { + fs := flag.NewFlagSet("create", flag.ExitOnError) + outputPath := fs.String("o", "output.eszip2", "Output file path") + checksum := fs.String("checksum", "sha256", "Checksum algorithm (none, sha256, xxhash3)") + fs.Usage = func() { + fmt.Println(`Usage: eszip create [options] + +Create a new eszip archive from files. + +Options:`) + fs.PrintDefaults() + fmt.Println(` +Examples: + eszip create -o app.eszip2 main.js utils.js + eszip create -checksum none -o app.eszip2 *.js`) + } + + fs.Parse(args) + if fs.NArg() < 1 { + fs.Usage() + os.Exit(1) + } + + archive := eszip.NewV2() + + // Set checksum + switch *checksum { + case "none": + archive.SetChecksum(eszip.ChecksumNone) + case "sha256": + archive.SetChecksum(eszip.ChecksumSha256) + case "xxhash3": + archive.SetChecksum(eszip.ChecksumXxh3) + default: + fmt.Fprintf(os.Stderr, "Unknown checksum: %s\n", *checksum) + os.Exit(1) + } + + // Add files + for _, filePath := range fs.Args() { + absPath, err := filepath.Abs(filePath) + if err != nil { + fmt.Fprintf(os.Stderr, "Error resolving path %s: %v\n", filePath, err) + os.Exit(1) + } + + content, err := os.ReadFile(absPath) + if err != nil { + fmt.Fprintf(os.Stderr, "Error reading file %s: %v\n", filePath, err) + os.Exit(1) + } + + // Determine module kind + kind := eszip.ModuleKindJavaScript + ext := strings.ToLower(filepath.Ext(filePath)) + switch ext { + case ".json": + kind = eszip.ModuleKindJson + case ".wasm": + kind = eszip.ModuleKindWasm + } + + specifier := "file://" + absPath + archive.AddModule(specifier, kind, content, nil) + fmt.Printf("Added: %s\n", specifier) + } + + // Serialize + data, err := archive.IntoBytes() + if err != nil { + fmt.Fprintf(os.Stderr, "Error serializing archive: %v\n", err) + os.Exit(1) + } + + // Write output + if err := os.WriteFile(*outputPath, data, 0644); err != nil { + fmt.Fprintf(os.Stderr, "Error writing output: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Created: %s (%d bytes)\n", *outputPath, len(data)) +} + +// infoCmd handles the 'info' command +func infoCmd(args []string) { + fs := flag.NewFlagSet("info", flag.ExitOnError) + fs.Usage = func() { + fmt.Println(`Usage: eszip info + +Show information about an eszip archive.`) + } + + fs.Parse(args) + if fs.NArg() < 1 { + fs.Usage() + os.Exit(1) + } + + archivePath := fs.Arg(0) + ctx := context.Background() + + // Get file size + stat, err := os.Stat(archivePath) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + + archive, err := loadArchive(ctx, archivePath) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + + specifiers := archive.Specifiers() + + fmt.Printf("File: %s\n", archivePath) + fmt.Printf("Size: %d bytes\n", stat.Size()) + + if archive.IsV1() { + fmt.Println("Format: V1 (JSON)") + } else { + fmt.Println("Format: V2 (binary)") + } + + fmt.Printf("Modules: %d\n", len(specifiers)) + + // Count by kind + kindCounts := make(map[eszip.ModuleKind]int) + redirectCount := 0 + totalSourceSize := 0 + + for _, spec := range specifiers { + module := archive.GetModule(spec) + if module == nil { + redirectCount++ + continue + } + kindCounts[module.Kind]++ + + source, _ := module.Source(ctx) + totalSourceSize += len(source) + } + + fmt.Println("\nModule types:") + for kind, count := range kindCounts { + fmt.Printf(" %s: %d\n", kind, count) + } + if redirectCount > 0 { + fmt.Printf(" redirects: %d\n", redirectCount) + } + + fmt.Printf("\nTotal source size: %d bytes\n", totalSourceSize) + + // Check for npm snapshot + if archive.IsV2() { + snapshot := archive.V2().TakeNpmSnapshot() + if snapshot != nil { + fmt.Printf("\nNPM packages: %d\n", len(snapshot.Packages)) + fmt.Printf("NPM root packages: %d\n", len(snapshot.RootPackages)) + } + } +} + +func loadArchive(ctx context.Context, path string) (*eszip.EszipUnion, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read file: %w", err) + } + + return eszip.ParseBytes(ctx, data) +} + +func specifierToPath(specifier string) string { + // Remove protocol prefixes + path := specifier + for _, prefix := range []string{"file:///", "file://", "https://", "http://"} { + if strings.HasPrefix(path, prefix) { + path = strings.TrimPrefix(path, prefix) + break + } + } + + // Clean the path + path = strings.TrimPrefix(path, "/") + + return path +} diff --git a/eszip-go/errors.go b/eszip-go/errors.go new file mode 100644 index 0000000..12016bd --- /dev/null +++ b/eszip-go/errors.go @@ -0,0 +1,112 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +package eszip + +import "fmt" + +// ParseErrorType represents the type of parse error +type ParseErrorType int + +const ( + ErrInvalidV1Json ParseErrorType = iota + ErrInvalidV1Version + ErrInvalidV2 + ErrInvalidV2HeaderHash + ErrInvalidV2Specifier + ErrInvalidV2EntryKind + ErrInvalidV2ModuleKind + ErrInvalidV2Header + ErrInvalidV2SourceOffset + ErrInvalidV2SourceHash + ErrInvalidV2NpmSnapshotHash + ErrInvalidV2NpmPackageOffset + ErrInvalidV2NpmPackage + ErrInvalidV2NpmPackageReq + ErrInvalidV22OptionsHeader + ErrInvalidV22OptionsHeaderHash + ErrIO +) + +// ParseError represents an error that occurred during parsing +type ParseError struct { + Type ParseErrorType + Message string + Offset int +} + +func (e *ParseError) Error() string { + if e.Offset > 0 { + return fmt.Sprintf("eszip parse error: %s at offset %d", e.Message, e.Offset) + } + return fmt.Sprintf("eszip parse error: %s", e.Message) +} + +// Error constructors for common parse errors + +func errInvalidV1Json(err error) *ParseError { + return &ParseError{Type: ErrInvalidV1Json, Message: fmt.Sprintf("invalid eszip v1 json: %v", err)} +} + +func errInvalidV1Version(version uint32) *ParseError { + return &ParseError{Type: ErrInvalidV1Version, Message: fmt.Sprintf("invalid eszip v1 version: got %d, expected 1", version)} +} + +func errInvalidV2() *ParseError { + return &ParseError{Type: ErrInvalidV2, Message: "invalid eszip v2"} +} + +func errInvalidV2HeaderHash() *ParseError { + return &ParseError{Type: ErrInvalidV2HeaderHash, Message: "invalid eszip v2 header hash"} +} + +func errInvalidV2Specifier(offset int) *ParseError { + return &ParseError{Type: ErrInvalidV2Specifier, Message: "invalid specifier in eszip v2 header", Offset: offset} +} + +func errInvalidV2EntryKind(kind uint8, offset int) *ParseError { + return &ParseError{Type: ErrInvalidV2EntryKind, Message: fmt.Sprintf("invalid entry kind %d in eszip v2 header", kind), Offset: offset} +} + +func errInvalidV2ModuleKind(kind uint8, offset int) *ParseError { + return &ParseError{Type: ErrInvalidV2ModuleKind, Message: fmt.Sprintf("invalid module kind %d in eszip v2 header", kind), Offset: offset} +} + +func errInvalidV2Header(msg string) *ParseError { + return &ParseError{Type: ErrInvalidV2Header, Message: fmt.Sprintf("invalid eszip v2 header: %s", msg)} +} + +func errInvalidV2SourceOffset(offset int) *ParseError { + return &ParseError{Type: ErrInvalidV2SourceOffset, Message: fmt.Sprintf("invalid eszip v2 source offset (%d)", offset), Offset: offset} +} + +func errInvalidV2SourceHash(specifier string) *ParseError { + return &ParseError{Type: ErrInvalidV2SourceHash, Message: fmt.Sprintf("invalid eszip v2 source hash (specifier %s)", specifier)} +} + +func errInvalidV2NpmSnapshotHash() *ParseError { + return &ParseError{Type: ErrInvalidV2NpmSnapshotHash, Message: "invalid eszip v2.1 npm snapshot hash"} +} + +func errInvalidV2NpmPackageOffset(index int, err error) *ParseError { + return &ParseError{Type: ErrInvalidV2NpmPackageOffset, Message: fmt.Sprintf("invalid eszip v2.1 npm package at index %d: %v", index, err)} +} + +func errInvalidV2NpmPackage(name string, err error) *ParseError { + return &ParseError{Type: ErrInvalidV2NpmPackage, Message: fmt.Sprintf("invalid eszip v2.1 npm package '%s': %v", name, err)} +} + +func errInvalidV2NpmPackageReq(req string, err error) *ParseError { + return &ParseError{Type: ErrInvalidV2NpmPackageReq, Message: fmt.Sprintf("invalid eszip v2.1 npm req '%s': %v", req, err)} +} + +func errInvalidV22OptionsHeader(msg string) *ParseError { + return &ParseError{Type: ErrInvalidV22OptionsHeader, Message: fmt.Sprintf("invalid eszip v2.2 options header: %s", msg)} +} + +func errInvalidV22OptionsHeaderHash() *ParseError { + return &ParseError{Type: ErrInvalidV22OptionsHeaderHash, Message: "invalid eszip v2.2 options header hash"} +} + +func errIO(err error) *ParseError { + return &ParseError{Type: ErrIO, Message: fmt.Sprintf("io error: %v", err)} +} diff --git a/eszip-go/eszip b/eszip-go/eszip new file mode 100755 index 0000000..45adb4e Binary files /dev/null and b/eszip-go/eszip differ diff --git a/eszip-go/eszip.go b/eszip-go/eszip.go new file mode 100644 index 0000000..d264109 --- /dev/null +++ b/eszip-go/eszip.go @@ -0,0 +1,175 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +// Package eszip provides functionality for reading and writing eszip archives. +// Eszip is a binary serialization format for ECMAScript module graphs, used by Deno. +package eszip + +import ( + "bufio" + "context" + "io" +) + +// Eszip is the unified interface for V1 and V2 eszip archives +type Eszip interface { + // GetModule returns the module for the given specifier, following redirects. + // Returns nil if not found or if the module is JSONC (use GetImportMap instead). + GetModule(specifier string) *Module + + // GetImportMap returns the import map module for the given specifier. + // Unlike GetModule, this can return JSONC modules. + GetImportMap(specifier string) *Module + + // Specifiers returns all module specifiers in the archive. + Specifiers() []string + + // TakeNpmSnapshot removes and returns the NPM resolution snapshot. + // Returns nil for V1 archives or if already taken. + TakeNpmSnapshot() *NpmResolutionSnapshot +} + +// EszipUnion wraps either V1 or V2 eszip +type EszipUnion struct { + v1 *EszipV1 + v2 *EszipV2 +} + +// IsV1 returns true if this is a V1 archive +func (e *EszipUnion) IsV1() bool { + return e.v1 != nil +} + +// IsV2 returns true if this is a V2 archive +func (e *EszipUnion) IsV2() bool { + return e.v2 != nil +} + +// V1 returns the V1 archive (panics if not V1) +func (e *EszipUnion) V1() *EszipV1 { + if e.v1 == nil { + panic("not a V1 eszip") + } + return e.v1 +} + +// V2 returns the V2 archive (panics if not V2) +func (e *EszipUnion) V2() *EszipV2 { + if e.v2 == nil { + panic("not a V2 eszip") + } + return e.v2 +} + +// GetModule returns the module for the given specifier +func (e *EszipUnion) GetModule(specifier string) *Module { + if e.v1 != nil { + return e.v1.GetModule(specifier) + } + return e.v2.GetModule(specifier) +} + +// GetImportMap returns the import map module for the given specifier +func (e *EszipUnion) GetImportMap(specifier string) *Module { + if e.v1 != nil { + return e.v1.GetImportMap(specifier) + } + return e.v2.GetImportMap(specifier) +} + +// Specifiers returns all module specifiers +func (e *EszipUnion) Specifiers() []string { + if e.v1 != nil { + return e.v1.Specifiers() + } + return e.v2.Specifiers() +} + +// TakeNpmSnapshot removes and returns the NPM snapshot +func (e *EszipUnion) TakeNpmSnapshot() *NpmResolutionSnapshot { + if e.v1 != nil { + return nil + } + return e.v2.TakeNpmSnapshot() +} + +// Parse parses an eszip archive from the given reader. +// Returns the eszip and a function to complete parsing of source data (for streaming). +// The completion function must be called to fully load sources. +func Parse(ctx context.Context, r io.Reader) (*EszipUnion, func(context.Context) error, error) { + br := bufio.NewReader(r) + + // Read magic bytes + magic := make([]byte, 8) + if _, err := io.ReadFull(br, magic); err != nil { + return nil, nil, errIO(err) + } + + // Check if it's V2 + if version, ok := VersionFromMagic(magic); ok { + eszip, complete, err := parseV2WithVersion(ctx, version, br) + if err != nil { + return nil, nil, err + } + return &EszipUnion{v2: eszip}, complete, nil + } + + // Otherwise, treat as V1 JSON - read the rest + var allData []byte + allData = append(allData, magic...) + remaining, err := io.ReadAll(br) + if err != nil { + return nil, nil, errIO(err) + } + allData = append(allData, remaining...) + + eszip, err := ParseV1(allData) + if err != nil { + return nil, nil, err + } + + // V1 has no streaming, completion is a no-op + complete := func(ctx context.Context) error { + return nil + } + + return &EszipUnion{v1: eszip}, complete, nil +} + +// ParseSync parses an eszip archive completely (blocking) +func ParseSync(ctx context.Context, r io.Reader) (*EszipUnion, error) { + eszip, complete, err := Parse(ctx, r) + if err != nil { + return nil, err + } + + if err := complete(ctx); err != nil { + return nil, err + } + + return eszip, nil +} + +// ParseBytes parses an eszip from a byte slice +func ParseBytes(ctx context.Context, data []byte) (*EszipUnion, error) { + return ParseSync(ctx, &byteReader{data: data}) +} + +// byteReader wraps a byte slice as an io.Reader +type byteReader struct { + data []byte + offset int +} + +func (r *byteReader) Read(p []byte) (n int, err error) { + if r.offset >= len(r.data) { + return 0, io.EOF + } + n = copy(p, r.data[r.offset:]) + r.offset += n + return n, nil +} + +// NewV2 creates a new empty V2 eszip archive +func NewV2() *EszipV2 { + return NewEszipV2() +} diff --git a/eszip-go/eszip_test.go b/eszip-go/eszip_test.go new file mode 100644 index 0000000..6e1d84b --- /dev/null +++ b/eszip-go/eszip_test.go @@ -0,0 +1,414 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +package eszip + +import ( + "bytes" + "context" + "os" + "testing" +) + +func TestParseV1(t *testing.T) { + data, err := os.ReadFile("testdata/basic.json") + if err != nil { + t.Fatalf("failed to read test file: %v", err) + } + + ctx := context.Background() + eszip, err := ParseBytes(ctx, data) + if err != nil { + t.Fatalf("failed to parse eszip: %v", err) + } + + if !eszip.IsV1() { + t.Fatal("expected V1 eszip") + } + + specifier := "https://gist.githubusercontent.com/lucacasonato/f3e21405322259ca4ed155722390fda2/raw/e25acb49b681e8e1da5a2a33744b7a36d538712d/hello.js" + module := eszip.GetModule(specifier) + if module == nil { + t.Fatalf("expected to find module: %s", specifier) + } + + if module.Specifier != specifier { + t.Errorf("expected specifier %s, got %s", specifier, module.Specifier) + } + + if module.Kind != ModuleKindJavaScript { + t.Errorf("expected JavaScript module, got %v", module.Kind) + } + + source, err := module.Source(ctx) + if err != nil { + t.Fatalf("failed to get source: %v", err) + } + + if len(source) == 0 { + t.Error("expected non-empty source") + } + + // Verify source contains expected content + if !bytes.Contains(source, []byte("Hello World")) { + t.Error("source should contain 'Hello World'") + } +} + +func TestParseV2(t *testing.T) { + data, err := os.ReadFile("testdata/redirect.eszip2") + if err != nil { + t.Fatalf("failed to read test file: %v", err) + } + + ctx := context.Background() + eszip, err := ParseBytes(ctx, data) + if err != nil { + t.Fatalf("failed to parse eszip: %v", err) + } + + if !eszip.IsV2() { + t.Fatal("expected V2 eszip") + } + + module := eszip.GetModule("file:///main.ts") + if module == nil { + t.Fatal("expected to find module: file:///main.ts") + } + + if module.Kind != ModuleKindJavaScript { + t.Errorf("expected JavaScript module, got %v", module.Kind) + } + + source, err := module.Source(ctx) + if err != nil { + t.Fatalf("failed to get source: %v", err) + } + + expectedSource := `export * as a from "./a.ts"; +` + if string(source) != expectedSource { + t.Errorf("expected source %q, got %q", expectedSource, string(source)) + } + + // Test source map + sourceMap, err := module.SourceMap(ctx) + if err != nil { + t.Fatalf("failed to get source map: %v", err) + } + + if len(sourceMap) == 0 { + t.Error("expected non-empty source map") + } +} + +func TestV2Redirect(t *testing.T) { + data, err := os.ReadFile("testdata/redirect.eszip2") + if err != nil { + t.Fatalf("failed to read test file: %v", err) + } + + ctx := context.Background() + eszip, err := ParseBytes(ctx, data) + if err != nil { + t.Fatalf("failed to parse eszip: %v", err) + } + + // file:///a.ts is a redirect to file:///b.ts + moduleA := eszip.GetModule("file:///a.ts") + if moduleA == nil { + t.Fatal("expected to find module: file:///a.ts") + } + + moduleB := eszip.GetModule("file:///b.ts") + if moduleB == nil { + t.Fatal("expected to find module: file:///b.ts") + } + + sourceA, _ := moduleA.Source(ctx) + sourceB, _ := moduleB.Source(ctx) + + // Both should have the same source since a.ts redirects to b.ts + if !bytes.Equal(sourceA, sourceB) { + t.Errorf("expected same source for redirect, got %q and %q", string(sourceA), string(sourceB)) + } +} + +func TestTakeSource(t *testing.T) { + data, err := os.ReadFile("testdata/basic.json") + if err != nil { + t.Fatalf("failed to read test file: %v", err) + } + + ctx := context.Background() + eszip, err := ParseBytes(ctx, data) + if err != nil { + t.Fatalf("failed to parse eszip: %v", err) + } + + specifier := "https://gist.githubusercontent.com/lucacasonato/f3e21405322259ca4ed155722390fda2/raw/e25acb49b681e8e1da5a2a33744b7a36d538712d/hello.js" + module := eszip.GetModule(specifier) + if module == nil { + t.Fatalf("expected to find module: %s", specifier) + } + + // Take the source + source, err := module.TakeSource(ctx) + if err != nil { + t.Fatalf("failed to take source: %v", err) + } + + if len(source) == 0 { + t.Error("expected non-empty source") + } + + // Module should no longer be available in V1 + module2 := eszip.GetModule(specifier) + if module2 != nil { + t.Error("expected module to be removed after take (V1 behavior)") + } +} + +func TestV2TakeSource(t *testing.T) { + data, err := os.ReadFile("testdata/redirect.eszip2") + if err != nil { + t.Fatalf("failed to read test file: %v", err) + } + + ctx := context.Background() + eszip, err := ParseBytes(ctx, data) + if err != nil { + t.Fatalf("failed to parse eszip: %v", err) + } + + module := eszip.GetModule("file:///main.ts") + if module == nil { + t.Fatal("expected to find module") + } + + // Take the source + source, err := module.TakeSource(ctx) + if err != nil { + t.Fatalf("failed to take source: %v", err) + } + + if len(source) == 0 { + t.Error("expected non-empty source") + } + + // Module should still be available but source should be nil + module2 := eszip.GetModule("file:///main.ts") + if module2 == nil { + t.Fatal("expected module to still exist in V2") + } + + source2, err := module2.Source(ctx) + if err != nil { + t.Fatalf("failed to get source: %v", err) + } + if source2 != nil { + t.Error("expected source to be nil after take") + } + + // Source map should still be available + sourceMap, err := module2.SourceMap(ctx) + if err != nil { + t.Fatalf("failed to get source map: %v", err) + } + if len(sourceMap) == 0 { + t.Error("expected source map to still be available") + } +} + +func TestV2Specifiers(t *testing.T) { + data, err := os.ReadFile("testdata/redirect.eszip2") + if err != nil { + t.Fatalf("failed to read test file: %v", err) + } + + ctx := context.Background() + eszip, err := ParseBytes(ctx, data) + if err != nil { + t.Fatalf("failed to parse eszip: %v", err) + } + + specs := eszip.Specifiers() + if len(specs) == 0 { + t.Error("expected at least one specifier") + } + + // Should contain main.ts, b.ts, and a.ts + expected := map[string]bool{ + "file:///main.ts": true, + "file:///b.ts": true, + "file:///a.ts": true, + } + + for _, spec := range specs { + delete(expected, spec) + } + + if len(expected) > 0 { + t.Errorf("missing specifiers: %v", expected) + } +} + +func TestNewV2AndWrite(t *testing.T) { + ctx := context.Background() + + // Create a new V2 eszip + eszip := NewV2() + + // Add a module + eszip.AddModule("file:///test.js", ModuleKindJavaScript, []byte("console.log('hello');"), []byte("{}")) + + // Add a redirect + eszip.AddRedirect("file:///alias.js", "file:///test.js") + + // Serialize + data, err := eszip.IntoBytes() + if err != nil { + t.Fatalf("failed to serialize eszip: %v", err) + } + + // Parse it back + parsed, err := ParseBytes(ctx, data) + if err != nil { + t.Fatalf("failed to parse serialized eszip: %v", err) + } + + if !parsed.IsV2() { + t.Fatal("expected V2 eszip") + } + + // Verify the module + module := parsed.GetModule("file:///test.js") + if module == nil { + t.Fatal("expected to find module") + } + + source, err := module.Source(ctx) + if err != nil { + t.Fatalf("failed to get source: %v", err) + } + + if string(source) != "console.log('hello');" { + t.Errorf("expected source %q, got %q", "console.log('hello');", string(source)) + } + + // Verify the redirect + aliasModule := parsed.GetModule("file:///alias.js") + if aliasModule == nil { + t.Fatal("expected to find alias module") + } + + aliasSource, err := aliasModule.Source(ctx) + if err != nil { + t.Fatalf("failed to get alias source: %v", err) + } + + if string(aliasSource) != "console.log('hello');" { + t.Errorf("expected alias source %q, got %q", "console.log('hello');", string(aliasSource)) + } +} + +func TestChecksumTypes(t *testing.T) { + testCases := []struct { + name string + checksum ChecksumType + }{ + {"NoChecksum", ChecksumNone}, + {"Sha256", ChecksumSha256}, + {"XxHash3", ChecksumXxh3}, + } + + ctx := context.Background() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + eszip := NewV2() + eszip.SetChecksum(tc.checksum) + eszip.AddModule("file:///test.js", ModuleKindJavaScript, []byte("test"), nil) + + data, err := eszip.IntoBytes() + if err != nil { + t.Fatalf("failed to serialize: %v", err) + } + + parsed, err := ParseBytes(ctx, data) + if err != nil { + t.Fatalf("failed to parse: %v", err) + } + + module := parsed.GetModule("file:///test.js") + if module == nil { + t.Fatal("expected to find module") + } + + source, err := module.Source(ctx) + if err != nil { + t.Fatalf("failed to get source: %v", err) + } + + if string(source) != "test" { + t.Errorf("expected source 'test', got %q", string(source)) + } + }) + } +} + +func TestModuleKinds(t *testing.T) { + testCases := []struct { + kind ModuleKind + name string + }{ + {ModuleKindJavaScript, "javascript"}, + {ModuleKindJson, "json"}, + {ModuleKindJsonc, "jsonc"}, + {ModuleKindOpaqueData, "opaque_data"}, + {ModuleKindWasm, "wasm"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.kind.String() != tc.name { + t.Errorf("expected %s, got %s", tc.name, tc.kind.String()) + } + }) + } +} + +func TestV1Iterator(t *testing.T) { + data, err := os.ReadFile("testdata/basic.json") + if err != nil { + t.Fatalf("failed to read test file: %v", err) + } + + eszip, err := ParseV1(data) + if err != nil { + t.Fatalf("failed to parse eszip: %v", err) + } + + modules := eszip.Iterate() + if len(modules) != 1 { + t.Errorf("expected 1 module, got %d", len(modules)) + } +} + +func TestV2Iterator(t *testing.T) { + data, err := os.ReadFile("testdata/redirect.eszip2") + if err != nil { + t.Fatalf("failed to read test file: %v", err) + } + + ctx := context.Background() + parsed, err := ParseBytes(ctx, data) + if err != nil { + t.Fatalf("failed to parse eszip: %v", err) + } + + modules := parsed.V2().Iterate() + // Should have 3 modules but only 2 are actual modules (one is redirect) + if len(modules) < 2 { + t.Errorf("expected at least 2 modules, got %d", len(modules)) + } +} diff --git a/eszip-go/go.mod b/eszip-go/go.mod new file mode 100644 index 0000000..83e9e0a --- /dev/null +++ b/eszip-go/go.mod @@ -0,0 +1,7 @@ +module github.com/example/eszip-go + +go 1.21 + +require github.com/zeebo/xxh3 v1.0.2 + +require github.com/klauspost/cpuid/v2 v2.0.9 // indirect diff --git a/eszip-go/go.sum b/eszip-go/go.sum new file mode 100644 index 0000000..fd6f1e4 --- /dev/null +++ b/eszip-go/go.sum @@ -0,0 +1,6 @@ +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= diff --git a/eszip-go/module.go b/eszip-go/module.go new file mode 100644 index 0000000..a27f2e4 --- /dev/null +++ b/eszip-go/module.go @@ -0,0 +1,201 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +package eszip + +import ( + "context" + "sync" +) + +// ModuleKind represents the type of module stored +type ModuleKind uint8 + +const ( + ModuleKindJavaScript ModuleKind = 0 + ModuleKindJson ModuleKind = 1 + ModuleKindJsonc ModuleKind = 2 + ModuleKindOpaqueData ModuleKind = 3 + ModuleKindWasm ModuleKind = 4 +) + +func (k ModuleKind) String() string { + switch k { + case ModuleKindJavaScript: + return "javascript" + case ModuleKindJson: + return "json" + case ModuleKindJsonc: + return "jsonc" + case ModuleKindOpaqueData: + return "opaque_data" + case ModuleKindWasm: + return "wasm" + default: + return "unknown" + } +} + +// Module represents a module in the eszip archive +type Module struct { + Specifier string + Kind ModuleKind + inner moduleInner +} + +// moduleInner provides access to module sources +type moduleInner interface { + getSource(ctx context.Context, specifier string) ([]byte, error) + takeSource(ctx context.Context, specifier string) ([]byte, error) + getSourceMap(ctx context.Context, specifier string) ([]byte, error) + takeSourceMap(ctx context.Context, specifier string) ([]byte, error) +} + +// Source returns the source code of the module. +// This may block if the source hasn't been loaded yet (streaming). +func (m *Module) Source(ctx context.Context) ([]byte, error) { + return m.inner.getSource(ctx, m.Specifier) +} + +// TakeSource returns and removes the source from memory. +func (m *Module) TakeSource(ctx context.Context) ([]byte, error) { + return m.inner.takeSource(ctx, m.Specifier) +} + +// SourceMap returns the source map of the module (V2 only). +func (m *Module) SourceMap(ctx context.Context) ([]byte, error) { + return m.inner.getSourceMap(ctx, m.Specifier) +} + +// TakeSourceMap returns and removes the source map from memory. +func (m *Module) TakeSourceMap(ctx context.Context) ([]byte, error) { + return m.inner.takeSourceMap(ctx, m.Specifier) +} + +// SourceSlotState represents the state of a source slot +type SourceSlotState int + +const ( + SourceSlotPending SourceSlotState = iota + SourceSlotReady + SourceSlotTaken +) + +// SourceSlot represents a pending or loaded source +type SourceSlot struct { + mu sync.RWMutex + state SourceSlotState + data []byte + offset uint32 + length uint32 + waitCh chan struct{} +} + +// NewPendingSourceSlot creates a new pending source slot +func NewPendingSourceSlot(offset, length uint32) *SourceSlot { + return &SourceSlot{ + state: SourceSlotPending, + offset: offset, + length: length, + waitCh: make(chan struct{}), + } +} + +// NewReadySourceSlot creates a new ready source slot with data +func NewReadySourceSlot(data []byte) *SourceSlot { + ch := make(chan struct{}) + close(ch) + return &SourceSlot{ + state: SourceSlotReady, + data: data, + waitCh: ch, + } +} + +// NewEmptySourceSlot creates a new ready source slot with empty data +func NewEmptySourceSlot() *SourceSlot { + return NewReadySourceSlot([]byte{}) +} + +// SetReady marks the slot as ready with the given data +func (s *SourceSlot) SetReady(data []byte) { + s.mu.Lock() + defer s.mu.Unlock() + s.data = data + s.state = SourceSlotReady + close(s.waitCh) +} + +// Get returns the source data, blocking until ready or context cancelled +func (s *SourceSlot) Get(ctx context.Context) ([]byte, error) { + s.mu.RLock() + if s.state == SourceSlotReady { + data := s.data + s.mu.RUnlock() + return data, nil + } + if s.state == SourceSlotTaken { + s.mu.RUnlock() + return nil, nil + } + waitCh := s.waitCh + s.mu.RUnlock() + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-waitCh: + s.mu.RLock() + defer s.mu.RUnlock() + if s.state == SourceSlotTaken { + return nil, nil + } + return s.data, nil + } +} + +// Take returns and removes the source data +func (s *SourceSlot) Take(ctx context.Context) ([]byte, error) { + s.mu.RLock() + if s.state == SourceSlotTaken { + s.mu.RUnlock() + return nil, nil + } + if s.state == SourceSlotPending { + waitCh := s.waitCh + s.mu.RUnlock() + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-waitCh: + } + } else { + s.mu.RUnlock() + } + + s.mu.Lock() + defer s.mu.Unlock() + if s.state == SourceSlotTaken { + return nil, nil + } + data := s.data + s.data = nil + s.state = SourceSlotTaken + return data, nil +} + +// State returns the current state +func (s *SourceSlot) State() SourceSlotState { + s.mu.RLock() + defer s.mu.RUnlock() + return s.state +} + +// Offset returns the offset in the sources section +func (s *SourceSlot) Offset() uint32 { + return s.offset +} + +// Length returns the length in the sources section +func (s *SourceSlot) Length() uint32 { + return s.length +} diff --git a/eszip-go/modulemap.go b/eszip-go/modulemap.go new file mode 100644 index 0000000..6c50853 --- /dev/null +++ b/eszip-go/modulemap.go @@ -0,0 +1,153 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +package eszip + +import ( + "sync" +) + +// ModuleMap is a thread-safe ordered map of modules +type ModuleMap struct { + mu sync.RWMutex + order []string + data map[string]EszipV2Module +} + +// EszipV2Module represents a module entry in V2 format +type EszipV2Module interface { + isEszipV2Module() +} + +// ModuleData represents an actual module with source +type ModuleData struct { + Kind ModuleKind + Source *SourceSlot + SourceMap *SourceSlot +} + +func (ModuleData) isEszipV2Module() {} + +// ModuleRedirect represents a redirect to another specifier +type ModuleRedirect struct { + Target string +} + +func (ModuleRedirect) isEszipV2Module() {} + +// NpmSpecifierEntry represents an npm specifier entry +type NpmSpecifierEntry struct { + PackageID uint32 +} + +func (NpmSpecifierEntry) isEszipV2Module() {} + +// NewModuleMap creates a new module map +func NewModuleMap() *ModuleMap { + return &ModuleMap{ + order: make([]string, 0), + data: make(map[string]EszipV2Module), + } +} + +// Insert adds or updates a module +func (m *ModuleMap) Insert(specifier string, module EszipV2Module) { + m.mu.Lock() + defer m.mu.Unlock() + if _, exists := m.data[specifier]; !exists { + m.order = append(m.order, specifier) + } + m.data[specifier] = module +} + +// InsertFront adds a module at the front (for import maps) +func (m *ModuleMap) InsertFront(specifier string, module EszipV2Module) { + m.mu.Lock() + defer m.mu.Unlock() + if _, exists := m.data[specifier]; exists { + // Remove from current position + for i, s := range m.order { + if s == specifier { + m.order = append(m.order[:i], m.order[i+1:]...) + break + } + } + } + m.order = append([]string{specifier}, m.order...) + m.data[specifier] = module +} + +// Get retrieves a module +func (m *ModuleMap) Get(specifier string) (EszipV2Module, bool) { + m.mu.RLock() + defer m.mu.RUnlock() + mod, ok := m.data[specifier] + return mod, ok +} + +// GetMut retrieves a module for mutation (returns the pointer) +func (m *ModuleMap) GetMut(specifier string) EszipV2Module { + m.mu.Lock() + defer m.mu.Unlock() + return m.data[specifier] +} + +// Remove removes a module and returns it +func (m *ModuleMap) Remove(specifier string) (EszipV2Module, bool) { + m.mu.Lock() + defer m.mu.Unlock() + mod, ok := m.data[specifier] + if ok { + delete(m.data, specifier) + for i, s := range m.order { + if s == specifier { + m.order = append(m.order[:i], m.order[i+1:]...) + break + } + } + } + return mod, ok +} + +// Keys returns all specifiers in order +func (m *ModuleMap) Keys() []string { + m.mu.RLock() + defer m.mu.RUnlock() + keys := make([]string, len(m.order)) + copy(keys, m.order) + return keys +} + +// Len returns the number of modules +func (m *ModuleMap) Len() int { + m.mu.RLock() + defer m.mu.RUnlock() + return len(m.order) +} + +// ModuleEntry represents a specifier-module pair for iteration +type ModuleEntry struct { + Specifier string + Module EszipV2Module +} + +// Iterate returns a channel that yields all modules +func (m *ModuleMap) Iterate() <-chan ModuleEntry { + ch := make(chan ModuleEntry) + go func() { + defer close(ch) + m.mu.RLock() + keys := make([]string, len(m.order)) + copy(keys, m.order) + m.mu.RUnlock() + + for _, key := range keys { + m.mu.RLock() + mod, ok := m.data[key] + m.mu.RUnlock() + if ok { + ch <- ModuleEntry{Specifier: key, Module: mod} + } + } + }() + return ch +} diff --git a/eszip-go/testdata/basic.json b/eszip-go/testdata/basic.json new file mode 100644 index 0000000..7ec06aa --- /dev/null +++ b/eszip-go/testdata/basic.json @@ -0,0 +1,13 @@ +{ + "version": 1, + "modules": { + "https://gist.githubusercontent.com/lucacasonato/f3e21405322259ca4ed155722390fda2/raw/e25acb49b681e8e1da5a2a33744b7a36d538712d/hello.js": { + "Source": { + "source": "addEventListener(\"fetch\", (event) => {\n event.respondWith(new Response(\"Hello World\", {\n headers: { \"content-type\": \"text/plain\" },\n }));\n});", + "transpiled": "addEventListener(\"fetch\", (event)=>{\n event.respondWith(new Response(\"Hello World\", {\n headers: {\n \"content-type\": \"text/plain\"\n }\n }));\n});\n//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIjxodHRwczovL2dpc3QuZ2l0aHVidXNlcmNvbnRlbnQuY29tL2x1Y2FjYXNvbmF0by9mM2UyMTQwNTMyMjI1OWNhNGVkMTU1NzIyMzkwZmRhMi9yYXcvZTI1YWNiNDliNjgxZThlMWRhNWEyYTMzNzQ0YjdhMzZkNTM4NzEyZC9oZWxsby5qcz4iXSwic291cmNlc0NvbnRlbnQiOlsiYWRkRXZlbnRMaXN0ZW5lcihcImZldGNoXCIsIChldmVudCkgPT4ge1xuICBldmVudC5yZXNwb25kV2l0aChuZXcgUmVzcG9uc2UoXCJIZWxsbyBXb3JsZFwiLCB7XG4gICAgaGVhZGVyczogeyBcImNvbnRlbnQtdHlwZVwiOiBcInRleHQvcGxhaW5cIiB9LFxuICB9KSk7XG59KTsiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsZ0JBQUEsRUFBQSxLQUFBLElBQUEsS0FBQTtBQUNBLFNBQUEsQ0FBQSxXQUFBLEtBQUEsUUFBQSxFQUFBLFdBQUE7QUFDQSxlQUFBO2FBQUEsWUFBQSxJQUFBLFVBQUEifQ==", + "content_type": "text/plain; charset=utf-8", + "deps": [] + } + } + } +} diff --git a/eszip-go/testdata/json.eszip2 b/eszip-go/testdata/json.eszip2 new file mode 100644 index 0000000..c2f73ef Binary files /dev/null and b/eszip-go/testdata/json.eszip2 differ diff --git a/eszip-go/testdata/redirect.eszip2 b/eszip-go/testdata/redirect.eszip2 new file mode 100644 index 0000000..b661e21 Binary files /dev/null and b/eszip-go/testdata/redirect.eszip2 differ diff --git a/eszip-go/testdata/wasm.eszip2_3 b/eszip-go/testdata/wasm.eszip2_3 new file mode 100644 index 0000000..d8e948f Binary files /dev/null and b/eszip-go/testdata/wasm.eszip2_3 differ diff --git a/eszip-go/v1.go b/eszip-go/v1.go new file mode 100644 index 0000000..af5553d --- /dev/null +++ b/eszip-go/v1.go @@ -0,0 +1,214 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +package eszip + +import ( + "context" + "encoding/json" + "net/url" + "sync" +) + +const eszipV1GraphVersion uint32 = 1 + +// EszipV1 represents a V1 eszip archive (JSON format) +type EszipV1 struct { + Version uint32 `json:"version"` + Modules map[string]json.RawMessage `json:"modules"` + + // Internal parsed modules + mu sync.RWMutex + parsedModules map[string]*moduleInfoV1 +} + +// moduleInfoV1 represents a module in V1 format +type moduleInfoV1 struct { + isRedirect bool + redirect string + source *moduleSourceV1 +} + +// moduleSourceV1 represents module source data +type moduleSourceV1 struct { + Source string `json:"source"` + Transpiled *string `json:"transpiled"` + ContentType *string `json:"content_type"` + Deps []string `json:"deps"` +} + +// v1ModuleInfoJSON is used for JSON unmarshaling +type v1ModuleInfoJSON struct { + Redirect *string `json:"Redirect"` + Source *moduleSourceV1 `json:"Source"` +} + +// ParseV1 parses a V1 eszip from JSON data +func ParseV1(data []byte) (*EszipV1, error) { + var eszip EszipV1 + if err := json.Unmarshal(data, &eszip); err != nil { + return nil, errInvalidV1Json(err) + } + + if eszip.Version != eszipV1GraphVersion { + return nil, errInvalidV1Version(eszip.Version) + } + + // Parse all modules + eszip.parsedModules = make(map[string]*moduleInfoV1) + for specifier, raw := range eszip.Modules { + var info v1ModuleInfoJSON + if err := json.Unmarshal(raw, &info); err != nil { + return nil, errInvalidV1Json(err) + } + + moduleInfo := &moduleInfoV1{} + if info.Redirect != nil { + moduleInfo.isRedirect = true + moduleInfo.redirect = *info.Redirect + } else if info.Source != nil { + moduleInfo.source = info.Source + } + eszip.parsedModules[specifier] = moduleInfo + } + + return &eszip, nil +} + +// GetModule returns the module for the given specifier, following redirects +func (e *EszipV1) GetModule(specifier string) *Module { + // Parse URL to normalize it + u, err := url.Parse(specifier) + if err != nil { + return nil + } + normalizedSpecifier := u.String() + + visited := make(map[string]bool) + current := normalizedSpecifier + + e.mu.RLock() + defer e.mu.RUnlock() + + for { + if visited[current] { + return nil // Cycle detected + } + visited[current] = true + + info, ok := e.parsedModules[current] + if !ok { + return nil + } + + if info.isRedirect { + current = info.redirect + continue + } + + return &Module{ + Specifier: current, + Kind: ModuleKindJavaScript, + inner: &v1ModuleInner{eszip: e}, + } + } +} + +// GetImportMap returns nil for V1 (V1 never contains import maps) +func (e *EszipV1) GetImportMap(specifier string) *Module { + return nil +} + +// Specifiers returns all module specifiers +func (e *EszipV1) Specifiers() []string { + e.mu.RLock() + defer e.mu.RUnlock() + + specs := make([]string, 0, len(e.parsedModules)) + for spec := range e.parsedModules { + specs = append(specs, spec) + } + return specs +} + +// IntoBytes serializes the V1 eszip to JSON +func (e *EszipV1) IntoBytes() ([]byte, error) { + return json.Marshal(e) +} + +// v1ModuleInner implements moduleInner for V1 +type v1ModuleInner struct { + eszip *EszipV1 +} + +func (v *v1ModuleInner) getSource(ctx context.Context, specifier string) ([]byte, error) { + v.eszip.mu.RLock() + defer v.eszip.mu.RUnlock() + + info, ok := v.eszip.parsedModules[specifier] + if !ok || info.isRedirect || info.source == nil { + return nil, nil + } + + // Return transpiled if available, otherwise source + if info.source.Transpiled != nil { + return []byte(*info.source.Transpiled), nil + } + return []byte(info.source.Source), nil +} + +func (v *v1ModuleInner) takeSource(ctx context.Context, specifier string) ([]byte, error) { + v.eszip.mu.Lock() + defer v.eszip.mu.Unlock() + + info, ok := v.eszip.parsedModules[specifier] + if !ok || info.isRedirect || info.source == nil { + return nil, nil + } + + // Get the source + var source []byte + if info.source.Transpiled != nil { + source = []byte(*info.source.Transpiled) + } else { + source = []byte(info.source.Source) + } + + // Remove the module from the map (V1 behavior) + delete(v.eszip.parsedModules, specifier) + + return source, nil +} + +func (v *v1ModuleInner) getSourceMap(ctx context.Context, specifier string) ([]byte, error) { + // V1 does not support source maps + return nil, nil +} + +func (v *v1ModuleInner) takeSourceMap(ctx context.Context, specifier string) ([]byte, error) { + // V1 does not support source maps + return nil, nil +} + +// Iterate returns all modules as an iterator +func (e *EszipV1) Iterate() []struct { + Specifier string + Module *Module +} { + specs := e.Specifiers() + result := make([]struct { + Specifier string + Module *Module + }, 0, len(specs)) + + for _, spec := range specs { + module := e.GetModule(spec) + if module != nil { + result = append(result, struct { + Specifier string + Module *Module + }{Specifier: spec, Module: module}) + } + } + + return result +} diff --git a/eszip-go/v2.go b/eszip-go/v2.go new file mode 100644 index 0000000..b3ac75a --- /dev/null +++ b/eszip-go/v2.go @@ -0,0 +1,370 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +package eszip + +import ( + "context" + "sync" +) + +// Magic bytes for V2 versions +var ( + MagicV2 = [8]byte{'E', 'S', 'Z', 'I', 'P', '_', 'V', '2'} + MagicV2_1 = [8]byte{'E', 'S', 'Z', 'I', 'P', '2', '.', '1'} + MagicV2_2 = [8]byte{'E', 'S', 'Z', 'I', 'P', '2', '.', '2'} + MagicV2_3 = [8]byte{'E', 'S', 'Z', 'I', 'P', '2', '.', '3'} +) + +// EszipVersion represents the V2 version +type EszipVersion int + +const ( + VersionV2 EszipVersion = 0 + VersionV2_1 EszipVersion = 1 + VersionV2_2 EszipVersion = 2 + VersionV2_3 EszipVersion = 3 +) + +// LatestVersion is the latest supported version +const LatestVersion = VersionV2_3 + +// VersionFromMagic returns the version from magic bytes +func VersionFromMagic(magic []byte) (EszipVersion, bool) { + if len(magic) < 8 { + return 0, false + } + var m [8]byte + copy(m[:], magic[:8]) + + switch m { + case MagicV2: + return VersionV2, true + case MagicV2_1: + return VersionV2_1, true + case MagicV2_2: + return VersionV2_2, true + case MagicV2_3: + return VersionV2_3, true + default: + return 0, false + } +} + +// ToMagic returns the magic bytes for the version +func (v EszipVersion) ToMagic() [8]byte { + switch v { + case VersionV2: + return MagicV2 + case VersionV2_1: + return MagicV2_1 + case VersionV2_2: + return MagicV2_2 + case VersionV2_3: + return MagicV2_3 + default: + return MagicV2_3 + } +} + +// SupportsNpm returns true if the version supports npm +func (v EszipVersion) SupportsNpm() bool { + return v != VersionV2 +} + +// SupportsOptions returns true if the version supports options header +func (v EszipVersion) SupportsOptions() bool { + return v >= VersionV2_2 +} + +// HeaderFrameKind represents the type of entry in the modules header +type HeaderFrameKind uint8 + +const ( + HeaderFrameModule HeaderFrameKind = 0 + HeaderFrameRedirect HeaderFrameKind = 1 + HeaderFrameNpmSpecifier HeaderFrameKind = 2 +) + +// Options represents V2 options +type Options struct { + Checksum ChecksumType + ChecksumSize uint8 +} + +// DefaultOptionsForVersion returns the default options for a version +func DefaultOptionsForVersion(version EszipVersion) Options { + opts := Options{ + Checksum: ChecksumNone, + } + // Versions prior to v2.2 default to SHA256 + if version == VersionV2 || version == VersionV2_1 { + opts.Checksum = ChecksumSha256 + } + opts.ChecksumSize = opts.Checksum.DigestSize() + return opts +} + +// GetChecksumSize returns the effective checksum size +func (o Options) GetChecksumSize() uint8 { + if o.ChecksumSize > 0 { + return o.ChecksumSize + } + return o.Checksum.DigestSize() +} + +// EszipV2 represents a V2 eszip archive +type EszipV2 struct { + modules *ModuleMap + npmSnapshot *NpmResolutionSnapshot + options Options + version EszipVersion +} + +// NewEszipV2 creates a new empty V2 eszip +func NewEszipV2() *EszipV2 { + return &EszipV2{ + modules: NewModuleMap(), + options: DefaultOptionsForVersion(LatestVersion), + version: LatestVersion, + } +} + +// HasMagic checks if the buffer starts with a V2 magic +func HasMagic(buffer []byte) bool { + if len(buffer) < 8 { + return false + } + _, ok := VersionFromMagic(buffer[:8]) + return ok +} + +// GetModule returns the module for the given specifier, following redirects +func (e *EszipV2) GetModule(specifier string) *Module { + return e.getModuleInternal(specifier, false) +} + +// GetImportMap returns the import map module for the given specifier +func (e *EszipV2) GetImportMap(specifier string) *Module { + return e.getModuleInternal(specifier, true) +} + +func (e *EszipV2) getModuleInternal(specifier string, allowJsonc bool) *Module { + visited := make(map[string]bool) + current := specifier + + for { + if visited[current] { + return nil // Cycle detected + } + visited[current] = true + + mod, ok := e.modules.Get(current) + if !ok { + return nil + } + + switch m := mod.(type) { + case *ModuleData: + if m.Kind == ModuleKindJsonc && !allowJsonc { + return nil + } + return &Module{ + Specifier: current, + Kind: m.Kind, + inner: &v2ModuleInner{eszip: e}, + } + case *ModuleRedirect: + current = m.Target + case *NpmSpecifierEntry: + // NPM specifiers are not regular modules + return nil + default: + return nil + } + } +} + +// Specifiers returns all module specifiers +func (e *EszipV2) Specifiers() []string { + return e.modules.Keys() +} + +// TakeNpmSnapshot removes and returns the NPM snapshot +func (e *EszipV2) TakeNpmSnapshot() *NpmResolutionSnapshot { + snapshot := e.npmSnapshot + e.npmSnapshot = nil + return snapshot +} + +// SetChecksum sets the checksum algorithm +func (e *EszipV2) SetChecksum(checksum ChecksumType) { + e.options.Checksum = checksum + e.options.ChecksumSize = checksum.DigestSize() +} + +// AddModule adds a module to the archive +func (e *EszipV2) AddModule(specifier string, kind ModuleKind, source, sourceMap []byte) { + e.modules.Insert(specifier, &ModuleData{ + Kind: kind, + Source: NewReadySourceSlot(source), + SourceMap: NewReadySourceSlot(sourceMap), + }) +} + +// AddImportMap adds an import map at the front of the archive +func (e *EszipV2) AddImportMap(kind ModuleKind, specifier string, source []byte) { + e.modules.InsertFront(specifier, &ModuleData{ + Kind: kind, + Source: NewReadySourceSlot(source), + SourceMap: NewEmptySourceSlot(), + }) +} + +// AddRedirect adds a redirect entry +func (e *EszipV2) AddRedirect(specifier, target string) { + e.modules.Insert(specifier, &ModuleRedirect{Target: target}) +} + +// AddOpaqueData adds opaque data to the archive +func (e *EszipV2) AddOpaqueData(specifier string, data []byte) { + e.AddModule(specifier, ModuleKindOpaqueData, data, nil) +} + +// Iterate returns all modules +func (e *EszipV2) Iterate() []struct { + Specifier string + Module *Module +} { + specs := e.Specifiers() + result := make([]struct { + Specifier string + Module *Module + }, 0, len(specs)) + + for _, spec := range specs { + module := e.GetModule(spec) + if module != nil { + result = append(result, struct { + Specifier string + Module *Module + }{Specifier: spec, Module: module}) + } + } + + return result +} + +// v2ModuleInner implements moduleInner for V2 +type v2ModuleInner struct { + eszip *EszipV2 +} + +func (v *v2ModuleInner) getSource(ctx context.Context, specifier string) ([]byte, error) { + mod, ok := v.eszip.modules.Get(specifier) + if !ok { + return nil, nil + } + + data, ok := mod.(*ModuleData) + if !ok { + return nil, nil + } + + return data.Source.Get(ctx) +} + +func (v *v2ModuleInner) takeSource(ctx context.Context, specifier string) ([]byte, error) { + mod, ok := v.eszip.modules.Get(specifier) + if !ok { + return nil, nil + } + + data, ok := mod.(*ModuleData) + if !ok { + return nil, nil + } + + return data.Source.Take(ctx) +} + +func (v *v2ModuleInner) getSourceMap(ctx context.Context, specifier string) ([]byte, error) { + mod, ok := v.eszip.modules.Get(specifier) + if !ok { + return nil, nil + } + + data, ok := mod.(*ModuleData) + if !ok { + return nil, nil + } + + return data.SourceMap.Get(ctx) +} + +func (v *v2ModuleInner) takeSourceMap(ctx context.Context, specifier string) ([]byte, error) { + mod, ok := v.eszip.modules.Get(specifier) + if !ok { + return nil, nil + } + + data, ok := mod.(*ModuleData) + if !ok { + return nil, nil + } + + return data.SourceMap.Take(ctx) +} + +// Section represents a parsed section with content and hash +type Section struct { + content []byte + hash []byte + checksum ChecksumType +} + +// Content returns the section content +func (s *Section) Content() []byte { + return s.content +} + +// ContentLen returns the content length +func (s *Section) ContentLen() int { + return len(s.content) +} + +// TotalLen returns the total length including hash +func (s *Section) TotalLen() int { + return len(s.content) + len(s.hash) +} + +// IsChecksumValid verifies the section checksum +func (s *Section) IsChecksumValid() bool { + if s.checksum == ChecksumNone { + return true + } + return s.checksum.Verify(s.content, s.hash) +} + +// IntoContent returns and takes ownership of the content +func (s *Section) IntoContent() []byte { + content := s.content + s.content = nil + return content +} + +// NpmPackageIndex represents an npm package index +type NpmPackageIndex struct { + Index uint32 +} + +// parserState holds state during parsing +type parserState struct { + mu sync.Mutex + sourceOffsets map[int]sourceOffsetEntry + sourceMapOffsets map[int]sourceOffsetEntry +} + +type sourceOffsetEntry struct { + length int + specifier string +} diff --git a/eszip-go/v2_npm.go b/eszip-go/v2_npm.go new file mode 100644 index 0000000..ab71dc8 --- /dev/null +++ b/eszip-go/v2_npm.go @@ -0,0 +1,187 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +package eszip + +import ( + "bufio" + "encoding/binary" + "fmt" + "strings" +) + +// NpmResolutionSnapshot represents the NPM package resolution +type NpmResolutionSnapshot struct { + Packages []*NpmPackage + RootPackages map[string]*NpmPackageID // req -> id +} + +// NpmPackage represents a resolved NPM package +type NpmPackage struct { + ID *NpmPackageID + Dependencies map[string]*NpmPackageID // req -> id +} + +// NpmPackageID represents an NPM package identifier (name@version) +type NpmPackageID struct { + Name string + Version string +} + +// String returns the serialized form of the package ID +func (id *NpmPackageID) String() string { + return fmt.Sprintf("%s@%s", id.Name, id.Version) +} + +// ParseNpmPackageID parses a serialized NPM package ID (name@version) +func ParseNpmPackageID(s string) (*NpmPackageID, error) { + // Find the last @ which separates name from version + // Package names can contain @ (like @types/node) + lastAt := strings.LastIndex(s, "@") + if lastAt <= 0 { + return nil, fmt.Errorf("invalid npm package id: %s", s) + } + + return &NpmPackageID{ + Name: s[:lastAt], + Version: s[lastAt+1:], + }, nil +} + +// parseNpmSection parses the NPM section +func parseNpmSection(br *bufio.Reader, options Options, npmSpecifiers map[string]NpmPackageIndex) (*NpmResolutionSnapshot, error) { + section, err := readSection(br, options) + if err != nil { + return nil, err + } + + if !section.IsChecksumValid() { + return nil, errInvalidV2NpmSnapshotHash() + } + + content := section.Content() + if len(content) == 0 { + return nil, nil + } + + // Parse packages + packages := make([]*npmModuleEntry, 0) + offset := 0 + + for offset < len(content) { + entry, newOffset, err := parseNpmModule(content, offset) + if err != nil { + return nil, errInvalidV2NpmPackageOffset(offset, err) + } + packages = append(packages, entry) + offset = newOffset + } + + // Build index to ID map + pkgIndexToID := make(map[uint32]*NpmPackageID) + for i, pkg := range packages { + id, err := ParseNpmPackageID(pkg.name) + if err != nil { + return nil, errInvalidV2NpmPackage(pkg.name, err) + } + pkgIndexToID[uint32(i)] = id + } + + // Build final packages + finalPackages := make([]*NpmPackage, 0, len(packages)) + for i, pkg := range packages { + id := pkgIndexToID[uint32(i)] + deps := make(map[string]*NpmPackageID) + + for req, idx := range pkg.dependencies { + depID, ok := pkgIndexToID[idx] + if !ok { + return nil, errInvalidV2NpmPackage(pkg.name, fmt.Errorf("missing index '%d'", idx)) + } + deps[req] = depID + } + + finalPackages = append(finalPackages, &NpmPackage{ + ID: id, + Dependencies: deps, + }) + } + + // Build root packages + rootPackages := make(map[string]*NpmPackageID) + for req, idx := range npmSpecifiers { + id, ok := pkgIndexToID[idx.Index] + if !ok { + return nil, errInvalidV2NpmPackageReq(req, fmt.Errorf("missing index '%d'", idx.Index)) + } + rootPackages[req] = id + } + + return &NpmResolutionSnapshot{ + Packages: finalPackages, + RootPackages: rootPackages, + }, nil +} + +// npmModuleEntry is an intermediate structure for parsing +type npmModuleEntry struct { + name string + dependencies map[string]uint32 // req -> package index +} + +func parseNpmModule(content []byte, offset int) (*npmModuleEntry, int, error) { + // Parse name + name, offset, err := parseNpmString(content, offset) + if err != nil { + return nil, 0, err + } + + // Parse dependency count + if offset+4 > len(content) { + return nil, 0, fmt.Errorf("unexpected end of data") + } + depCount := binary.BigEndian.Uint32(content[offset : offset+4]) + offset += 4 + + // Parse dependencies + deps := make(map[string]uint32) + for i := uint32(0); i < depCount; i++ { + // Parse dependency name + depName, newOffset, err := parseNpmString(content, offset) + if err != nil { + return nil, 0, err + } + offset = newOffset + + // Parse package index + if offset+4 > len(content) { + return nil, 0, fmt.Errorf("unexpected end of data") + } + pkgIndex := binary.BigEndian.Uint32(content[offset : offset+4]) + offset += 4 + + deps[depName] = pkgIndex + } + + return &npmModuleEntry{ + name: name, + dependencies: deps, + }, offset, nil +} + +func parseNpmString(content []byte, offset int) (string, int, error) { + if offset+4 > len(content) { + return "", 0, fmt.Errorf("unexpected end of data") + } + + length := binary.BigEndian.Uint32(content[offset : offset+4]) + offset += 4 + + if offset+int(length) > len(content) { + return "", 0, fmt.Errorf("unexpected end of data") + } + + str := string(content[offset : offset+int(length)]) + offset += int(length) + + return str, offset, nil +} diff --git a/eszip-go/v2_parser.go b/eszip-go/v2_parser.go new file mode 100644 index 0000000..5ea1ac4 --- /dev/null +++ b/eszip-go/v2_parser.go @@ -0,0 +1,444 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +package eszip + +import ( + "bufio" + "context" + "encoding/binary" + "io" +) + +// ParseV2 parses a V2 eszip from a reader. +// Returns the eszip and a completion function that loads sources in background. +func ParseV2(ctx context.Context, r io.Reader) (*EszipV2, func(context.Context) error, error) { + br := bufio.NewReader(r) + + // Read magic bytes + magic := make([]byte, 8) + if _, err := io.ReadFull(br, magic); err != nil { + return nil, nil, errIO(err) + } + + version, ok := VersionFromMagic(magic) + if !ok { + return nil, nil, errInvalidV2() + } + + return parseV2WithVersion(ctx, version, br) +} + +// ParseV2Sync parses a V2 eszip completely (blocking) +func ParseV2Sync(ctx context.Context, r io.Reader) (*EszipV2, error) { + eszip, complete, err := ParseV2(ctx, r) + if err != nil { + return nil, err + } + + if err := complete(ctx); err != nil { + return nil, err + } + + return eszip, nil +} + +func parseV2WithVersion(ctx context.Context, version EszipVersion, br *bufio.Reader) (*EszipV2, func(context.Context) error, error) { + supportsNpm := version.SupportsNpm() + supportsOptions := version.SupportsOptions() + + options := DefaultOptionsForVersion(version) + + // Parse options header (V2.2+) + if supportsOptions { + var err error + options, err = parseOptionsHeader(br, options) + if err != nil { + return nil, nil, err + } + } + + // Parse modules header + modulesHeader, err := readSection(br, options) + if err != nil { + return nil, nil, err + } + + if !modulesHeader.IsChecksumValid() { + return nil, nil, errInvalidV2HeaderHash() + } + + // Parse module entries from header + modules, npmSpecifiers, err := parseModulesHeader(modulesHeader.Content(), supportsNpm) + if err != nil { + return nil, nil, err + } + + // Parse NPM section + var npmSnapshot *NpmResolutionSnapshot + if supportsNpm { + npmSnapshot, err = parseNpmSection(br, options, npmSpecifiers) + if err != nil { + return nil, nil, err + } + } + + // Build source offset maps + sourceOffsets := make(map[int]sourceOffsetEntry) + sourceMapOffsets := make(map[int]sourceOffsetEntry) + + for _, specifier := range modules.Keys() { + mod, ok := modules.Get(specifier) + if !ok { + continue + } + + data, ok := mod.(*ModuleData) + if !ok { + continue + } + + if data.Source.State() == SourceSlotPending && data.Source.Length() > 0 { + sourceOffsets[int(data.Source.Offset())] = sourceOffsetEntry{ + length: int(data.Source.Length()), + specifier: specifier, + } + } + + if data.SourceMap.State() == SourceSlotPending && data.SourceMap.Length() > 0 { + sourceMapOffsets[int(data.SourceMap.Offset())] = sourceOffsetEntry{ + length: int(data.SourceMap.Length()), + specifier: specifier, + } + } + } + + eszip := &EszipV2{ + modules: modules, + npmSnapshot: npmSnapshot, + options: options, + version: version, + } + + // Return completion function for source loading + completeFn := func(ctx context.Context) error { + return loadSources(ctx, br, eszip, options, sourceOffsets, sourceMapOffsets) + } + + return eszip, completeFn, nil +} + +func parseOptionsHeader(br *bufio.Reader, defaults Options) (Options, error) { + // Read options without checksum first + preOpts := defaults + preOpts.Checksum = ChecksumNone + preOpts.ChecksumSize = 0 + + optionsHeader, err := readSection(br, preOpts) + if err != nil { + return defaults, err + } + + if optionsHeader.ContentLen()%2 != 0 { + return defaults, errInvalidV22OptionsHeader("options are expected to be byte tuples") + } + + options := defaults + content := optionsHeader.Content() + + for i := 0; i < len(content); i += 2 { + option := content[i] + value := content[i+1] + + switch option { + case 0: // Checksum type + checksum, ok := ChecksumFromU8(value) + if ok { + options.Checksum = checksum + } + case 1: // Checksum size + options.ChecksumSize = value + } + // Unknown options are ignored for forward compatibility + } + + if options.GetChecksumSize() == 0 && options.Checksum != ChecksumNone { + return defaults, errInvalidV22OptionsHeader("checksum size must be known") + } + + // If checksum is enabled, validate the options header hash + if options.GetChecksumSize() > 0 { + // Read the hash that follows + hash := make([]byte, options.GetChecksumSize()) + if _, err := io.ReadFull(br, hash); err != nil { + return defaults, errIO(err) + } + + if !options.Checksum.Verify(content, hash) { + return defaults, errInvalidV22OptionsHeaderHash() + } + } + + return options, nil +} + +func readSection(br *bufio.Reader, options Options) (*Section, error) { + // Read length (4 bytes, big-endian) + lengthBytes := make([]byte, 4) + if _, err := io.ReadFull(br, lengthBytes); err != nil { + return nil, errIO(err) + } + length := binary.BigEndian.Uint32(lengthBytes) + + // Read content + content := make([]byte, length) + if _, err := io.ReadFull(br, content); err != nil { + return nil, errIO(err) + } + + // Read hash + checksumSize := options.GetChecksumSize() + var hash []byte + if checksumSize > 0 { + hash = make([]byte, checksumSize) + if _, err := io.ReadFull(br, hash); err != nil { + return nil, errIO(err) + } + } + + return &Section{ + content: content, + hash: hash, + checksum: options.Checksum, + }, nil +} + +func readSectionWithSize(br *bufio.Reader, options Options, contentLen int) (*Section, error) { + // Read content + content := make([]byte, contentLen) + if _, err := io.ReadFull(br, content); err != nil { + return nil, errIO(err) + } + + // Read hash + checksumSize := options.GetChecksumSize() + var hash []byte + if checksumSize > 0 { + hash = make([]byte, checksumSize) + if _, err := io.ReadFull(br, hash); err != nil { + return nil, errIO(err) + } + } + + return &Section{ + content: content, + hash: hash, + checksum: options.Checksum, + }, nil +} + +func parseModulesHeader(content []byte, supportsNpm bool) (*ModuleMap, map[string]NpmPackageIndex, error) { + modules := NewModuleMap() + npmSpecifiers := make(map[string]NpmPackageIndex) + + read := 0 + + for read < len(content) { + // Read specifier length + if read+4 > len(content) { + return nil, nil, errInvalidV2Header("specifier len") + } + specifierLen := int(binary.BigEndian.Uint32(content[read : read+4])) + read += 4 + + // Read specifier + if read+specifierLen > len(content) { + return nil, nil, errInvalidV2Header("specifier") + } + specifier := string(content[read : read+specifierLen]) + read += specifierLen + + // Read entry kind + if read+1 > len(content) { + return nil, nil, errInvalidV2Header("entry kind") + } + entryKind := content[read] + read++ + + switch entryKind { + case 0: // Module + if read+17 > len(content) { + return nil, nil, errInvalidV2Header("module data") + } + + sourceOffset := binary.BigEndian.Uint32(content[read : read+4]) + read += 4 + sourceLen := binary.BigEndian.Uint32(content[read : read+4]) + read += 4 + sourceMapOffset := binary.BigEndian.Uint32(content[read : read+4]) + read += 4 + sourceMapLen := binary.BigEndian.Uint32(content[read : read+4]) + read += 4 + kindByte := content[read] + read++ + + var kind ModuleKind + switch kindByte { + case 0: + kind = ModuleKindJavaScript + case 1: + kind = ModuleKindJson + case 2: + kind = ModuleKindJsonc + case 3: + kind = ModuleKindOpaqueData + case 4: + kind = ModuleKindWasm + default: + return nil, nil, errInvalidV2ModuleKind(kindByte, read) + } + + var source *SourceSlot + if sourceOffset == 0 && sourceLen == 0 { + source = NewEmptySourceSlot() + } else { + source = NewPendingSourceSlot(sourceOffset, sourceLen) + } + + var sourceMap *SourceSlot + if sourceMapOffset == 0 && sourceMapLen == 0 { + sourceMap = NewEmptySourceSlot() + } else { + sourceMap = NewPendingSourceSlot(sourceMapOffset, sourceMapLen) + } + + modules.Insert(specifier, &ModuleData{ + Kind: kind, + Source: source, + SourceMap: sourceMap, + }) + + case 1: // Redirect + if read+4 > len(content) { + return nil, nil, errInvalidV2Header("target len") + } + targetLen := int(binary.BigEndian.Uint32(content[read : read+4])) + read += 4 + + if read+targetLen > len(content) { + return nil, nil, errInvalidV2Header("target") + } + target := string(content[read : read+targetLen]) + read += targetLen + + modules.Insert(specifier, &ModuleRedirect{Target: target}) + + case 2: // NpmSpecifier + if !supportsNpm { + return nil, nil, errInvalidV2EntryKind(entryKind, read) + } + + if read+4 > len(content) { + return nil, nil, errInvalidV2Header("npm package id") + } + pkgID := binary.BigEndian.Uint32(content[read : read+4]) + read += 4 + + npmSpecifiers[specifier] = NpmPackageIndex{Index: pkgID} + + default: + return nil, nil, errInvalidV2EntryKind(entryKind, read) + } + } + + return modules, npmSpecifiers, nil +} + +func loadSources(ctx context.Context, br *bufio.Reader, eszip *EszipV2, options Options, sourceOffsets, sourceMapOffsets map[int]sourceOffsetEntry) error { + // Read sources section + sourcesLenBytes := make([]byte, 4) + if _, err := io.ReadFull(br, sourcesLenBytes); err != nil { + return errIO(err) + } + sourcesLen := int(binary.BigEndian.Uint32(sourcesLenBytes)) + + read := 0 + for read < sourcesLen { + entry, ok := sourceOffsets[read] + if !ok { + return errInvalidV2SourceOffset(read) + } + + section, err := readSectionWithSize(br, options, entry.length) + if err != nil { + return err + } + + if !section.IsChecksumValid() { + return errInvalidV2SourceHash(entry.specifier) + } + + read += section.TotalLen() + + // Update the module's source slot + mod, ok := eszip.modules.Get(entry.specifier) + if !ok { + continue + } + + data, ok := mod.(*ModuleData) + if !ok { + continue + } + + data.Source.SetReady(section.IntoContent()) + } + + // Read source maps section + sourceMapsLenBytes := make([]byte, 4) + if _, err := io.ReadFull(br, sourceMapsLenBytes); err != nil { + return errIO(err) + } + sourceMapsLen := int(binary.BigEndian.Uint32(sourceMapsLenBytes)) + + read = 0 + for read < sourceMapsLen { + entry, ok := sourceMapOffsets[read] + if !ok { + return errInvalidV2SourceOffset(read) + } + + section, err := readSectionWithSize(br, options, entry.length) + if err != nil { + return err + } + + if !section.IsChecksumValid() { + return errInvalidV2SourceHash(entry.specifier) + } + + read += section.TotalLen() + + // Update the module's source map slot + mod, ok := eszip.modules.Get(entry.specifier) + if !ok { + continue + } + + data, ok := mod.(*ModuleData) + if !ok { + continue + } + + data.SourceMap.SetReady(section.IntoContent()) + } + + return nil +} + +func readU32(r io.Reader) (uint32, error) { + buf := make([]byte, 4) + if _, err := io.ReadFull(r, buf); err != nil { + return 0, err + } + return binary.BigEndian.Uint32(buf), nil +} diff --git a/eszip-go/v2_writer.go b/eszip-go/v2_writer.go new file mode 100644 index 0000000..64b460b --- /dev/null +++ b/eszip-go/v2_writer.go @@ -0,0 +1,223 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +package eszip + +import ( + "encoding/binary" + "sort" +) + +// IntoBytes serializes the eszip archive to bytes +func (e *EszipV2) IntoBytes() ([]byte, error) { + checksum := e.options.Checksum + checksumSize := e.options.GetChecksumSize() + + var result []byte + + // Write magic (latest version) + magic := LatestVersion.ToMagic() + result = append(result, magic[:]...) + + // Build options header + optionsHeaderContent := []byte{ + 0, byte(checksum), // Checksum type + 1, byte(checksumSize), // Checksum size + } + + // Write options header length + optionsHeaderLenBytes := make([]byte, 4) + binary.BigEndian.PutUint32(optionsHeaderLenBytes, uint32(len(optionsHeaderContent))) + result = append(result, optionsHeaderLenBytes...) + + // Write options header content + result = append(result, optionsHeaderContent...) + + // Write options header hash + optionsHash := checksum.Hash(optionsHeaderContent) + result = append(result, optionsHash...) + + // Build modules header, sources, and source maps + var modulesHeader []byte + var sources []byte + var sourceMaps []byte + + keys := e.modules.Keys() + for _, specifier := range keys { + mod, ok := e.modules.Get(specifier) + if !ok { + continue + } + + // Write specifier + appendString(&modulesHeader, specifier) + + switch m := mod.(type) { + case *ModuleData: + // Write module entry + modulesHeader = append(modulesHeader, byte(HeaderFrameModule)) + + // Get source bytes + sourceBytes := m.Source.data + if sourceBytes == nil && m.Source.State() == SourceSlotReady { + sourceBytes = []byte{} + } + sourceLen := uint32(len(sourceBytes)) + + if sourceLen > 0 { + sourceOffset := uint32(len(sources)) + sources = append(sources, sourceBytes...) + sources = append(sources, checksum.Hash(sourceBytes)...) + + modulesHeader = appendU32BE(modulesHeader, sourceOffset) + modulesHeader = appendU32BE(modulesHeader, sourceLen) + } else { + modulesHeader = appendU32BE(modulesHeader, 0) + modulesHeader = appendU32BE(modulesHeader, 0) + } + + // Get source map bytes + sourceMapBytes := m.SourceMap.data + if sourceMapBytes == nil && m.SourceMap.State() == SourceSlotReady { + sourceMapBytes = []byte{} + } + sourceMapLen := uint32(len(sourceMapBytes)) + + if sourceMapLen > 0 { + sourceMapOffset := uint32(len(sourceMaps)) + sourceMaps = append(sourceMaps, sourceMapBytes...) + sourceMaps = append(sourceMaps, checksum.Hash(sourceMapBytes)...) + + modulesHeader = appendU32BE(modulesHeader, sourceMapOffset) + modulesHeader = appendU32BE(modulesHeader, sourceMapLen) + } else { + modulesHeader = appendU32BE(modulesHeader, 0) + modulesHeader = appendU32BE(modulesHeader, 0) + } + + // Write module kind + modulesHeader = append(modulesHeader, byte(m.Kind)) + + case *ModuleRedirect: + // Write redirect entry + modulesHeader = append(modulesHeader, byte(HeaderFrameRedirect)) + appendString(&modulesHeader, m.Target) + + case *NpmSpecifierEntry: + // Write npm specifier entry + modulesHeader = append(modulesHeader, byte(HeaderFrameNpmSpecifier)) + modulesHeader = appendU32BE(modulesHeader, m.PackageID) + } + } + + // Add npm snapshot entries if present + var npmBytes []byte + if e.npmSnapshot != nil { + // Sort packages by ID for determinism + packages := make([]*NpmPackage, len(e.npmSnapshot.Packages)) + copy(packages, e.npmSnapshot.Packages) + sort.Slice(packages, func(i, j int) bool { + return packages[i].ID.String() < packages[j].ID.String() + }) + + // Build ID to index map + idToIndex := make(map[string]uint32) + for i, pkg := range packages { + idToIndex[pkg.ID.String()] = uint32(i) + } + + // Write root packages to modules header + rootPkgs := make([]struct { + req string + id string + }, 0, len(e.npmSnapshot.RootPackages)) + for req, id := range e.npmSnapshot.RootPackages { + rootPkgs = append(rootPkgs, struct { + req string + id string + }{req: req, id: id.String()}) + } + sort.Slice(rootPkgs, func(i, j int) bool { + return rootPkgs[i].req < rootPkgs[j].req + }) + + for _, rp := range rootPkgs { + appendString(&modulesHeader, rp.req) + modulesHeader = append(modulesHeader, byte(HeaderFrameNpmSpecifier)) + modulesHeader = appendU32BE(modulesHeader, idToIndex[rp.id]) + } + + // Write packages to npm bytes + for _, pkg := range packages { + appendString(&npmBytes, pkg.ID.String()) + + // Write dependencies count + npmBytes = appendU32BE(npmBytes, uint32(len(pkg.Dependencies))) + + // Sort dependencies for determinism + deps := make([]struct { + req string + id string + }, 0, len(pkg.Dependencies)) + for req, id := range pkg.Dependencies { + deps = append(deps, struct { + req string + id string + }{req: req, id: id.String()}) + } + sort.Slice(deps, func(i, j int) bool { + return deps[i].req < deps[j].req + }) + + for _, dep := range deps { + appendString(&npmBytes, dep.req) + npmBytes = appendU32BE(npmBytes, idToIndex[dep.id]) + } + } + } + + // Write modules header length + modulesHeaderLenBytes := make([]byte, 4) + binary.BigEndian.PutUint32(modulesHeaderLenBytes, uint32(len(modulesHeader))) + result = append(result, modulesHeaderLenBytes...) + + // Write modules header content + result = append(result, modulesHeader...) + + // Write modules header hash + modulesHash := checksum.Hash(modulesHeader) + result = append(result, modulesHash...) + + // Write npm section + npmLenBytes := make([]byte, 4) + binary.BigEndian.PutUint32(npmLenBytes, uint32(len(npmBytes))) + result = append(result, npmLenBytes...) + result = append(result, npmBytes...) + result = append(result, checksum.Hash(npmBytes)...) + + // Write sources section + sourcesLenBytes := make([]byte, 4) + binary.BigEndian.PutUint32(sourcesLenBytes, uint32(len(sources))) + result = append(result, sourcesLenBytes...) + result = append(result, sources...) + + // Write source maps section + sourceMapsLenBytes := make([]byte, 4) + binary.BigEndian.PutUint32(sourceMapsLenBytes, uint32(len(sourceMaps))) + result = append(result, sourceMapsLenBytes...) + result = append(result, sourceMaps...) + + return result, nil +} + +func appendString(buf *[]byte, s string) { + lenBytes := make([]byte, 4) + binary.BigEndian.PutUint32(lenBytes, uint32(len(s))) + *buf = append(*buf, lenBytes...) + *buf = append(*buf, []byte(s)...) +} + +func appendU32BE(buf []byte, v uint32) []byte { + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, v) + return append(buf, b...) +}