package git import ( "context" "encoding/binary" "os" "os/exec" "path/filepath" "testing" "unicode/utf16" ) func utf16Bytes(s string, order binary.ByteOrder) []byte { units := utf16.Encode([]rune(s)) out := make([]byte, 0, len(units)*2) for _, u := range units { var pair [2]byte order.PutUint16(pair[:], u) out = append(out, pair[:]...) } return out } func TestNormalizeTextEncoding(t *testing.T) { utf8BOM := append([]byte{0xEF, 0xBB, 0xBF}, []byte("# readme")...) if got := string(normalizeTextEncoding(utf8BOM)); got != "# readme" { t.Fatalf("utf-8 BOM: got %q", got) } le := append([]byte{0xFF, 0xFE}, utf16Bytes("# readme \u2713", binary.LittleEndian)...) if got := string(normalizeTextEncoding(le)); got != "# readme \u2713" { t.Fatalf("utf-16le: got %q", got) } be := append([]byte{0xFE, 0xFF}, utf16Bytes("# readme \u2713", binary.BigEndian)...) if got := string(normalizeTextEncoding(be)); got != "# readme \u2713" { t.Fatalf("utf-16be: got %q", got) } plain := []byte("# plain readme") if got := string(normalizeTextEncoding(plain)); got != "# plain readme" { t.Fatalf("plain text must stay untouched, got %q", got) } binaryData := []byte{0x00, 0x01, 0x02, 0x03} if got := normalizeTextEncoding(binaryData); len(got) != len(binaryData) { t.Fatalf("binary data must stay untouched") } } // TestReadBlobDecodesUTF16 mirrors README files written by Windows tooling // (PowerShell writes UTF-16LE by default) and expects them to render as text. func TestReadBlobDecodesUTF16(t *testing.T) { ctx := context.Background() tmp := t.TempDir() barePath := filepath.Join(tmp, "repo.git") if err := InitBare(ctx, barePath, "main"); err != nil { t.Fatalf("InitBare: %v", err) } work := filepath.Join(tmp, "work") if err := os.MkdirAll(work, 0o755); err != nil { t.Fatalf("mkdir: %v", err) } run := func(args ...string) { t.Helper() cmd := exec.CommandContext(ctx, "git", args...) cmd.Dir = work if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v: %v (%s)", args, err, string(out)) } } run("init", "-b", "main") run("config", "user.name", "Tester") run("config", "user.email", "tester@example.com") content := append([]byte{0xFF, 0xFE}, utf16Bytes("# Windows README\n", binary.LittleEndian)...) if err := os.WriteFile(filepath.Join(work, "README.md"), content, 0o644); err != nil { t.Fatalf("write README: %v", err) } run("add", ".") run("commit", "-m", "utf-16 readme") run("push", barePath, "main") blob, err := ReadBlob(ctx, barePath, "main", "README.md") if err != nil { t.Fatalf("ReadBlob: %v", err) } if blob.IsBinary { t.Fatalf("BOM-marked UTF-16 file must not be treated as binary") } if string(blob.Data) != "# Windows README\n" { t.Fatalf("unexpected decoded content %q", string(blob.Data)) } readme, err := FindReadme(ctx, barePath, "main", "") if err != nil || readme == nil { t.Fatalf("FindReadme should find the UTF-16 README, err=%v", err) } }