cli,wasi: passes mounts directly as preopens to run rust wasi-testsuite (#1078)

This allows wasi to see mounts as individual preopens instead of
virtualized under a root. (GOOS=js doesn't use pre-opens so it still
sees a virtual root). This allows us to progress wasi-testsuite rust
tests, but acknowledges a current glitch in zig support, which is
tracked separately as #1077

Signed-off-by: Adrian Cole <adrian@tetrate.io>

.github/workflows/examples.yaml

@@ -83,7 +83,7 @@ jobs:
           toolchain: stable
           target: wasm32-wasi
 
-      - name: Build TinyGO examples
+      - name: Build TinyGo examples
         run: make build.examples.tinygo
 
       - name: Build AssemblyScript examples

.github/workflows/integration.yaml

@@ -13,6 +13,10 @@ on:
       - 'site/**'
       - 'netlify.toml'
 
+defaults:
+  run:  # use bash for all operating systems unless overridden
+    shell: bash
+
 env:  # Update this prior to requiring a higher minor version in go.mod
   GO_VERSION: "1.19"  # 1.xx == latest patch of 1.xx
   ZIG_BUILD_VERSION: "0.11.0-dev.725+9bcfe55b5"
@@ -177,12 +181,17 @@ jobs:
         os: [ubuntu-20.04, macos-12] #todo: windows-2022
 
     steps:
+      # TODO: remove cargo/rust parts after https://github.com/WebAssembly/wasi-testsuite/issues/49
       - uses: actions/cache@v3
         id: cache
         with:
           path:
+            ~/.cargo
             ~/.cache/go-build
-          key: integration-test-wasi-testsuite-${{ env.GO_VERSION }}
+            ~/go/pkg/mod
+            ~/.rustup/toolchains/
+            tests/rust/target
+          key: integration-test-wasi-testsuite-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum', '**/Cargo.lock', '**/Cargo.toml', '**/*.rs') }}
 
       - uses: actions/setup-go@v3
         with:
@@ -211,12 +220,28 @@ jobs:
       - name: Install dependencies
         working-directory: test-runner
         run: |
-          pip install -r requirements/dev.txt
+          python3 -m pip install -r requirements.txt
+
+      # TODO: Remove after https://github.com/WebAssembly/wasi-testsuite/issues/49
+      - name: Install wasm32-wasi target
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: wasm32-wasi
+
+      # TODO: Remove after https://github.com/WebAssembly/wasi-testsuite/issues/49
+      - name: Compile rust tests
+        working-directory: tests/rust
+        run: |
+          ./build.sh
 
       - name: Run all wasi-testsuite
         run: |
               python3 test-runner/wasi_test_runner.py \
                 -t ./tests/assemblyscript/testsuite/ \
                 ./tests/c/testsuite/ \
+                ./tests/rust/testsuite/ \
                 -r adapters/wazero.sh
+        # TODO: remove when #1036 is complete
+        continue-on-error: true
 

cmd/wazero/wazero_test.go

@@ -266,14 +266,14 @@ func TestRun(t *testing.T) {
 			wazeroOpts: []string{"--hostlogging=filesystem", fmt.Sprintf("--mount=%s:/animals:ro", bearDir)},
 			wasmArgs:   []string{"/animals/bear.txt"},
 			expectedStderr: fmt.Sprintf(`==> wasi_snapshot_preview1.fd_prestat_get(fd=3)
-<== (prestat={pr_name_len=1},errno=ESUCCESS)
+<== (prestat={pr_name_len=8},errno=ESUCCESS)
 ==> wasi_snapshot_preview1.fd_prestat_dir_name(fd=3)
-<== (path=/,errno=ESUCCESS)
+<== (path=/animals,errno=ESUCCESS)
 ==> wasi_snapshot_preview1.fd_prestat_get(fd=4)
 <== (prestat=,errno=EBADF)
 ==> wasi_snapshot_preview1.fd_fdstat_get(fd=3)
 <== (stat={filetype=DIRECTORY,fdflags=,fs_rights_base=,fs_rights_inheriting=},errno=ESUCCESS)
-==> wasi_snapshot_preview1.path_open(fd=3,dirflags=SYMLINK_FOLLOW,path=animals/bear.txt,oflags=,fs_rights_base=,fs_rights_inheriting=,fdflags=)
+==> wasi_snapshot_preview1.path_open(fd=3,dirflags=SYMLINK_FOLLOW,path=bear.txt,oflags=,fs_rights_base=,fs_rights_inheriting=,fdflags=)
 <== (opened_fd=4,errno=ESUCCESS)
 ==> wasi_snapshot_preview1.fd_filestat_get(fd=4)
 <== (filestat={filetype=REGULAR_FILE,size=5,mtim=%d},errno=ESUCCESS)

imports/wasi_snapshot_preview1/example/cat_test.go

@@ -59,6 +59,11 @@ func Test_cli(t *testing.T) {
 		tt := tc
 		t.Run(tt.toolchain, func(t *testing.T) {
 			for _, testPath := range []string{"/test.txt", "/testcases/test.txt"} {
+				if tt.toolchain == "zig" && testPath == "/testcases/test.txt" {
+					// Zig only resolves absolute paths under the first
+					// pre-open (cwd), so it won't find this file until #1077
+					continue
+				}
 				t.Run(testPath, func(t *testing.T) {
 					// Write out embedded files instead of accessing directly for docker cross-architecture tests.
 					wasmPath := filepath.Join(t.TempDir(), "cat.wasm")

imports/wasi_snapshot_preview1/wasi_stdlib_test.go

@@ -147,6 +147,7 @@ func testPreopen(t *testing.T, bin []byte) {
 1: stdout
 2: stderr
 3: /
+4: /tmp
 `, "\n"+console)
 }
 

internal/sys/fs.go

@@ -226,14 +226,26 @@ func NewFSContext(stdin io.Reader, stdout, stderr io.Writer, rootFS sysfs.FS) (f
 		return fsc, nil
 	}
 
-	// TODO: destructure CompositeFS into multiple pre-opens after #1077
-	fsc.openedFiles.Insert(&FileEntry{
-		FS:          rootFS,
-		Name:        "/",
-		IsPreopen:   true,
-		isDirectory: true,
-		File:        &lazyDir{fs: rootFS},
-	})
+	if comp, ok := rootFS.(*sysfs.CompositeFS); ok {
+		preopens := comp.FS()
+		for i, p := range comp.GuestPaths() {
+			fsc.openedFiles.Insert(&FileEntry{
+				FS:          preopens[i],
+				Name:        p,
+				IsPreopen:   true,
+				isDirectory: true,
+				File:        &lazyDir{fs: rootFS},
+			})
+		}
+	} else {
+		fsc.openedFiles.Insert(&FileEntry{
+			FS:          rootFS,
+			Name:        "/",
+			IsPreopen:   true,
+			isDirectory: true,
+			File:        &lazyDir{fs: rootFS},
+		})
+	}
 
 	return fsc, nil
 }

internal/sys/fs_test.go

@@ -111,6 +111,36 @@ func TestUnimplementedFSContext(t *testing.T) {
 	})
 }
 
+func TestCompositeFSContext(t *testing.T) {
+	tmpDir1 := t.TempDir()
+	testFS1 := sysfs.NewDirFS(tmpDir1)
+
+	tmpDir2 := t.TempDir()
+	testFS2 := sysfs.NewDirFS(tmpDir2)
+
+	rootFS, err := sysfs.NewRootFS([]sysfs.FS{testFS2, testFS1}, []string{"/tmp", "/"})
+	require.NoError(t, err)
+
+	testFS, err := NewFSContext(nil, nil, nil, rootFS)
+	require.NoError(t, err)
+
+	// Ensure the pre-opens have exactly the name specified, and are in order.
+	preopen3, ok := testFS.openedFiles.Lookup(3)
+	require.True(t, ok)
+	require.Equal(t, "/tmp", preopen3.Name)
+	preopen4, ok := testFS.openedFiles.Lookup(4)
+	require.True(t, ok)
+	require.Equal(t, "/", preopen4.Name)
+
+	t.Run("Close closes", func(t *testing.T) {
+		err := testFS.Close(testCtx)
+		require.NoError(t, err)
+
+		// Closes opened files
+		require.Equal(t, &FSContext{rootFS: rootFS}, testFS)
+	})
+}
+
 func TestContext_Close(t *testing.T) {
 	testFS := sysfs.Adapt(testfs.FS{"foo": &testfs.File{}})