os: implement CopyFS

Fixes #62484

Change-Id: I5d8950dedf86af48f42a641940b34e62aa2cddcb
Reviewed-on: https://go-review.googlesource.com/c/go/+/558995
Auto-Submit: Ian Lance Taylor <iant@google.com>
Reviewed-by: Ian Lance Taylor <iant@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>

Author: panjf2000

api/next/62484.txt

@@ -0,0 +1 @@
+pkg os, func CopyFS(string, fs.FS) error #62484

doc/next/6-stdlib/99-minor/os/62484.md

@@ -0,0 +1,2 @@
+The [`CopyFS`](/os#CopyFS) function copies an [`io/fs.FS`](/io/fs#FS)
+into the local filesystem.

src/os/dir.go

@@ -5,6 +5,8 @@
 package os
 
 import (
+	"internal/safefilepath"
+	"io"
 	"io/fs"
 	"sort"
 )
@@ -123,3 +125,61 @@ func ReadDir(name string) ([]DirEntry, error) {
 	sort.Slice(dirs, func(i, j int) bool { return dirs[i].Name() < dirs[j].Name() })
 	return dirs, err
 }
+
+// CopyFS copies the file system fsys into the directory dir,
+// creating dir if necessary.
+//
+// Newly created directories and files have their default modes
+// where any bits from the file in fsys that are not part of the
+// standard read, write, and execute permissions will be zeroed
+// out, and standard read and write permissions are set for owner,
+// group, and others while retaining any existing execute bits from
+// the file in fsys.
+//
+// Symbolic links in fsys are not supported, a *PathError with Err set
+// to ErrInvalid is returned on symlink.
+//
+// Copying stops at and returns the first error encountered.
+func CopyFS(dir string, fsys fs.FS) error {
+	return fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		fpath, err := safefilepath.FromFS(path)
+		if err != nil {
+			return err
+		}
+		newPath := joinPath(dir, fpath)
+		if d.IsDir() {
+			return MkdirAll(newPath, 0777)
+		}
+
+		// TODO(panjf2000): handle symlinks with the help of fs.ReadLinkFS
+		// 		once https://go.dev/issue/49580 is done.
+		//		we also need safefilepath.IsLocal from https://go.dev/cl/564295.
+		if !d.Type().IsRegular() {
+			return &PathError{Op: "CopyFS", Path: path, Err: ErrInvalid}
+		}
+
+		r, err := fsys.Open(path)
+		if err != nil {
+			return err
+		}
+		defer r.Close()
+		info, err := r.Stat()
+		if err != nil {
+			return err
+		}
+		w, err := OpenFile(newPath, O_CREATE|O_TRUNC|O_WRONLY, 0666|info.Mode()&0777)
+		if err != nil {
+			return err
+		}
+
+		if _, err := io.Copy(w, r); err != nil {
+			w.Close()
+			return &PathError{Op: "Copy", Path: newPath, Err: err}
+		}
+		return w.Close()
+	})
+}

src/os/os_test.go

@@ -5,6 +5,7 @@
 package os_test
 
 import (
+	"bytes"
 	"errors"
 	"flag"
 	"fmt"
@@ -3030,35 +3031,44 @@ func TestOpenFileKeepsPermissions(t *testing.T) {
 	}
 }
 
-func TestDirFS(t *testing.T) {
-	t.Parallel()
+func forceMFTUpdateOnWindows(t *testing.T, path string) {
+	t.Helper()
+
+	if runtime.GOOS != "windows" {
+		return
+	}
 
 	// On Windows, we force the MFT to update by reading the actual metadata from GetFileInformationByHandle and then
 	// explicitly setting that. Otherwise it might get out of sync with FindFirstFile. See golang.org/issues/42637.
-	if runtime.GOOS == "windows" {
-		if err := filepath.WalkDir("./testdata/dirfs", func(path string, d fs.DirEntry, err error) error {
-			if err != nil {
-				t.Fatal(err)
-			}
-			info, err := d.Info()
-			if err != nil {
-				t.Fatal(err)
-			}
-			stat, err := Stat(path) // This uses GetFileInformationByHandle internally.
-			if err != nil {
-				t.Fatal(err)
-			}
-			if stat.ModTime() == info.ModTime() {
-				return nil
-			}
-			if err := Chtimes(path, stat.ModTime(), stat.ModTime()); err != nil {
-				t.Log(err) // We only log, not die, in case the test directory is not writable.
-			}
-			return nil
-		}); err != nil {
+	if err := filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			t.Fatal(err)
+		}
+		info, err := d.Info()
+		if err != nil {
 			t.Fatal(err)
 		}
+		stat, err := Stat(path) // This uses GetFileInformationByHandle internally.
+		if err != nil {
+			t.Fatal(err)
+		}
+		if stat.ModTime() == info.ModTime() {
+			return nil
+		}
+		if err := Chtimes(path, stat.ModTime(), stat.ModTime()); err != nil {
+			t.Log(err) // We only log, not die, in case the test directory is not writable.
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
 	}
+}
+
+func TestDirFS(t *testing.T) {
+	t.Parallel()
+
+	forceMFTUpdateOnWindows(t, "./testdata/dirfs")
+
 	fsys := DirFS("./testdata/dirfs")
 	if err := fstest.TestFS(fsys, "a", "b", "dir/x"); err != nil {
 		t.Fatal(err)
@@ -3358,3 +3368,223 @@ func TestRandomLen(t *testing.T) {
 		}
 	}
 }
+
+func TestCopyFS(t *testing.T) {
+	t.Parallel()
+
+	// Test with disk filesystem.
+	forceMFTUpdateOnWindows(t, "./testdata/dirfs")
+	fsys := DirFS("./testdata/dirfs")
+	tmpDir := t.TempDir()
+	if err := CopyFS(tmpDir, fsys); err != nil {
+		t.Fatal("CopyFS:", err)
+	}
+	forceMFTUpdateOnWindows(t, tmpDir)
+	tmpFsys := DirFS(tmpDir)
+	if err := fstest.TestFS(tmpFsys, "a", "b", "dir/x"); err != nil {
+		t.Fatal("TestFS:", err)
+	}
+	if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
+		if d.IsDir() {
+			return nil
+		}
+
+		data, err := fs.ReadFile(fsys, path)
+		if err != nil {
+			return err
+		}
+		newData, err := fs.ReadFile(tmpFsys, path)
+		if err != nil {
+			return err
+		}
+		if !bytes.Equal(data, newData) {
+			return errors.New("file " + path + " contents differ")
+		}
+		return nil
+	}); err != nil {
+		t.Fatal("comparing two directories:", err)
+	}
+
+	// Test with memory filesystem.
+	fsys = fstest.MapFS{
+		"william":    {Data: []byte("Shakespeare\n")},
+		"carl":       {Data: []byte("Gauss\n")},
+		"daVinci":    {Data: []byte("Leonardo\n")},
+		"einstein":   {Data: []byte("Albert\n")},
+		"dir/newton": {Data: []byte("Sir Isaac\n")},
+	}
+	tmpDir = t.TempDir()
+	if err := CopyFS(tmpDir, fsys); err != nil {
+		t.Fatal("CopyFS:", err)
+	}
+	forceMFTUpdateOnWindows(t, tmpDir)
+	tmpFsys = DirFS(tmpDir)
+	if err := fstest.TestFS(tmpFsys, "william", "carl", "daVinci", "einstein", "dir/newton"); err != nil {
+		t.Fatal("TestFS:", err)
+	}
+	if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
+		if d.IsDir() {
+			return nil
+		}
+
+		data, err := fs.ReadFile(fsys, path)
+		if err != nil {
+			return err
+		}
+		newData, err := fs.ReadFile(tmpFsys, path)
+		if err != nil {
+			return err
+		}
+		if !bytes.Equal(data, newData) {
+			return errors.New("file " + path + " contents differ")
+		}
+		return nil
+	}); err != nil {
+		t.Fatal("comparing two directories:", err)
+	}
+}
+
+func TestCopyFSWithSymlinks(t *testing.T) {
+	// Test it with absolute and relative symlinks that point inside and outside the tree.
+	testenv.MustHaveSymlink(t)
+
+	// Create a directory and file outside.
+	tmpDir := t.TempDir()
+	outsideDir, err := MkdirTemp(tmpDir, "copyfs")
+	if err != nil {
+		t.Fatalf("MkdirTemp: %v", err)
+	}
+	outsideFile := filepath.Join(outsideDir, "file.out.txt")
+
+	if err := WriteFile(outsideFile, []byte("Testing CopyFS outside"), 0644); err != nil {
+		t.Fatalf("WriteFile: %v", err)
+	}
+
+	// Create a directory and file inside.
+	testDataDir, err := filepath.Abs("./testdata/")
+	if err != nil {
+		t.Fatalf("filepath.Abs: %v", err)
+	}
+	insideDir := filepath.Join(testDataDir, "copyfs")
+	if err := Mkdir(insideDir, 0755); err != nil {
+		t.Fatalf("Mkdir: %v", err)
+	}
+	defer RemoveAll(insideDir)
+	insideFile := filepath.Join(insideDir, "file.in.txt")
+	if err := WriteFile(insideFile, []byte("Testing CopyFS inside"), 0644); err != nil {
+		t.Fatalf("WriteFile: %v", err)
+	}
+
+	// Create directories for symlinks.
+	linkInDir := filepath.Join(insideDir, "in_symlinks")
+	if err := Mkdir(linkInDir, 0755); err != nil {
+		t.Fatalf("Mkdir: %v", err)
+	}
+	linkOutDir := filepath.Join(insideDir, "out_symlinks")
+	if err := Mkdir(linkOutDir, 0755); err != nil {
+		t.Fatalf("Mkdir: %v", err)
+	}
+
+	// First, we create the absolute symlink pointing outside.
+	outLinkFile := filepath.Join(linkOutDir, "file.abs.out.link")
+	if err := Symlink(outsideFile, outLinkFile); err != nil {
+		t.Fatalf("Symlink: %v", err)
+	}
+
+	// Then, we create the relative symlink pointing outside.
+	relOutsideFile, err := filepath.Rel(filepath.Join(linkOutDir, "."), outsideFile)
+	if err != nil {
+		t.Fatalf("filepath.Rel: %v", err)
+	}
+	relOutLinkFile := filepath.Join(linkOutDir, "file.rel.out.link")
+	if err := Symlink(relOutsideFile, relOutLinkFile); err != nil {
+		t.Fatalf("Symlink: %v", err)
+	}
+
+	// Last, we create the relative symlink pointing inside.
+	relInsideFile, err := filepath.Rel(filepath.Join(linkInDir, "."), insideFile)
+	if err != nil {
+		t.Fatalf("filepath.Rel: %v", err)
+	}
+	relInLinkFile := filepath.Join(linkInDir, "file.rel.in.link")
+	if err := Symlink(relInsideFile, relInLinkFile); err != nil {
+		t.Fatalf("Symlink: %v", err)
+	}
+
+	// Copy the directory tree and verify.
+	forceMFTUpdateOnWindows(t, insideDir)
+	fsys := DirFS(insideDir)
+	tmpDupDir, err := MkdirTemp(tmpDir, "copyfs_dup")
+	if err != nil {
+		t.Fatalf("MkdirTemp: %v", err)
+	}
+
+	// TODO(panjf2000): symlinks are currently not supported, and a specific error
+	// 			will be returned. Verify that error and skip the subsequent test,
+	//			revisit this once #49580 is closed.
+	if err := CopyFS(tmpDupDir, fsys); !errors.Is(err, ErrInvalid) {
+		t.Fatalf("got %v, want ErrInvalid", err)
+	}
+	t.Skip("skip the subsequent test and wait for #49580")
+
+	forceMFTUpdateOnWindows(t, tmpDupDir)
+	tmpFsys := DirFS(tmpDupDir)
+	if err := fstest.TestFS(tmpFsys, "file.in.txt", "out_symlinks/file.abs.out.link", "out_symlinks/file.rel.out.link", "in_symlinks/file.rel.in.link"); err != nil {
+		t.Fatal("TestFS:", err)
+	}
+	if err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
+		if d.IsDir() {
+			return nil
+		}
+
+		fi, err := d.Info()
+		if err != nil {
+			return err
+		}
+		if filepath.Ext(path) == ".link" {
+			if fi.Mode()&ModeSymlink == 0 {
+				return errors.New("original file " + path + " should be a symlink")
+			}
+			tmpfi, err := fs.Stat(tmpFsys, path)
+			if err != nil {
+				return err
+			}
+			if tmpfi.Mode()&ModeSymlink != 0 {
+				return errors.New("copied file " + path + " should not be a symlink")
+			}
+		}
+
+		data, err := fs.ReadFile(fsys, path)
+		if err != nil {
+			return err
+		}
+		newData, err := fs.ReadFile(tmpFsys, path)
+		if err != nil {
+			return err
+		}
+		if !bytes.Equal(data, newData) {
+			return errors.New("file " + path + " contents differ")
+		}
+
+		var target string
+		switch fileName := filepath.Base(path); fileName {
+		case "file.abs.out.link", "file.rel.out.link":
+			target = outsideFile
+		case "file.rel.in.link":
+			target = insideFile
+		}
+		if len(target) > 0 {
+			targetData, err := ReadFile(target)
+			if err != nil {
+				return err
+			}
+			if !bytes.Equal(targetData, newData) {
+				return errors.New("file " + path + " contents differ from target")
+			}
+		}
+
+		return nil
+	}); err != nil {
+		t.Fatal("comparing two directories:", err)
+	}
+}