| package main |
| |
| import ( |
| "archive/tar" |
| "bytes" |
| "encoding/json" |
| "fmt" |
| "io" |
| "io/ioutil" |
| "net/http" |
| "regexp" |
| "strings" |
| |
| "github.com/docker/docker/api/types" |
| "github.com/docker/docker/integration-cli/checker" |
| "github.com/docker/docker/integration-cli/cli/build/fakecontext" |
| "github.com/docker/docker/integration-cli/cli/build/fakegit" |
| "github.com/docker/docker/integration-cli/cli/build/fakestorage" |
| "github.com/docker/docker/integration-cli/request" |
| "github.com/go-check/check" |
| "github.com/moby/buildkit/session" |
| "github.com/moby/buildkit/session/filesync" |
| "github.com/stretchr/testify/assert" |
| "github.com/stretchr/testify/require" |
| "golang.org/x/net/context" |
| "golang.org/x/sync/errgroup" |
| ) |
| |
| func (s *DockerSuite) TestBuildAPIDockerFileRemote(c *check.C) { |
| testRequires(c, NotUserNamespace) |
| |
| var testD string |
| if testEnv.DaemonPlatform() == "windows" { |
| testD = `FROM busybox |
| RUN find / -name ba* |
| RUN find /tmp/` |
| } else { |
| // -xdev is required because sysfs can cause EPERM |
| testD = `FROM busybox |
| RUN find / -xdev -name ba* |
| RUN find /tmp/` |
| } |
| server := fakestorage.New(c, "", fakecontext.WithFiles(map[string]string{"testD": testD})) |
| defer server.Close() |
| |
| res, body, err := request.Post("/build?dockerfile=baz&remote="+server.URL()+"/testD", request.JSON) |
| c.Assert(err, checker.IsNil) |
| c.Assert(res.StatusCode, checker.Equals, http.StatusOK) |
| |
| buf, err := request.ReadBody(body) |
| c.Assert(err, checker.IsNil) |
| |
| // Make sure Dockerfile exists. |
| // Make sure 'baz' doesn't exist ANYWHERE despite being mentioned in the URL |
| out := string(buf) |
| c.Assert(out, checker.Contains, "RUN find /tmp") |
| c.Assert(out, checker.Not(checker.Contains), "baz") |
| } |
| |
| func (s *DockerSuite) TestBuildAPIRemoteTarballContext(c *check.C) { |
| buffer := new(bytes.Buffer) |
| tw := tar.NewWriter(buffer) |
| defer tw.Close() |
| |
| dockerfile := []byte("FROM busybox") |
| err := tw.WriteHeader(&tar.Header{ |
| Name: "Dockerfile", |
| Size: int64(len(dockerfile)), |
| }) |
| // failed to write tar file header |
| c.Assert(err, checker.IsNil) |
| |
| _, err = tw.Write(dockerfile) |
| // failed to write tar file content |
| c.Assert(err, checker.IsNil) |
| |
| // failed to close tar archive |
| c.Assert(tw.Close(), checker.IsNil) |
| |
| server := fakestorage.New(c, "", fakecontext.WithBinaryFiles(map[string]*bytes.Buffer{ |
| "testT.tar": buffer, |
| })) |
| defer server.Close() |
| |
| res, b, err := request.Post("/build?remote="+server.URL()+"/testT.tar", request.ContentType("application/tar")) |
| c.Assert(err, checker.IsNil) |
| c.Assert(res.StatusCode, checker.Equals, http.StatusOK) |
| b.Close() |
| } |
| |
| func (s *DockerSuite) TestBuildAPIRemoteTarballContextWithCustomDockerfile(c *check.C) { |
| buffer := new(bytes.Buffer) |
| tw := tar.NewWriter(buffer) |
| defer tw.Close() |
| |
| dockerfile := []byte(`FROM busybox |
| RUN echo 'wrong'`) |
| err := tw.WriteHeader(&tar.Header{ |
| Name: "Dockerfile", |
| Size: int64(len(dockerfile)), |
| }) |
| // failed to write tar file header |
| c.Assert(err, checker.IsNil) |
| |
| _, err = tw.Write(dockerfile) |
| // failed to write tar file content |
| c.Assert(err, checker.IsNil) |
| |
| custom := []byte(`FROM busybox |
| RUN echo 'right' |
| `) |
| err = tw.WriteHeader(&tar.Header{ |
| Name: "custom", |
| Size: int64(len(custom)), |
| }) |
| |
| // failed to write tar file header |
| c.Assert(err, checker.IsNil) |
| |
| _, err = tw.Write(custom) |
| // failed to write tar file content |
| c.Assert(err, checker.IsNil) |
| |
| // failed to close tar archive |
| c.Assert(tw.Close(), checker.IsNil) |
| |
| server := fakestorage.New(c, "", fakecontext.WithBinaryFiles(map[string]*bytes.Buffer{ |
| "testT.tar": buffer, |
| })) |
| defer server.Close() |
| |
| url := "/build?dockerfile=custom&remote=" + server.URL() + "/testT.tar" |
| res, body, err := request.Post(url, request.ContentType("application/tar")) |
| c.Assert(err, checker.IsNil) |
| c.Assert(res.StatusCode, checker.Equals, http.StatusOK) |
| |
| defer body.Close() |
| content, err := request.ReadBody(body) |
| c.Assert(err, checker.IsNil) |
| |
| // Build used the wrong dockerfile. |
| c.Assert(string(content), checker.Not(checker.Contains), "wrong") |
| } |
| |
| func (s *DockerSuite) TestBuildAPILowerDockerfile(c *check.C) { |
| git := fakegit.New(c, "repo", map[string]string{ |
| "dockerfile": `FROM busybox |
| RUN echo from dockerfile`, |
| }, false) |
| defer git.Close() |
| |
| res, body, err := request.Post("/build?remote="+git.RepoURL, request.JSON) |
| c.Assert(err, checker.IsNil) |
| c.Assert(res.StatusCode, checker.Equals, http.StatusOK) |
| |
| buf, err := request.ReadBody(body) |
| c.Assert(err, checker.IsNil) |
| |
| out := string(buf) |
| c.Assert(out, checker.Contains, "from dockerfile") |
| } |
| |
| func (s *DockerSuite) TestBuildAPIBuildGitWithF(c *check.C) { |
| git := fakegit.New(c, "repo", map[string]string{ |
| "baz": `FROM busybox |
| RUN echo from baz`, |
| "Dockerfile": `FROM busybox |
| RUN echo from Dockerfile`, |
| }, false) |
| defer git.Close() |
| |
| // Make sure it tries to 'dockerfile' query param value |
| res, body, err := request.Post("/build?dockerfile=baz&remote="+git.RepoURL, request.JSON) |
| c.Assert(err, checker.IsNil) |
| c.Assert(res.StatusCode, checker.Equals, http.StatusOK) |
| |
| buf, err := request.ReadBody(body) |
| c.Assert(err, checker.IsNil) |
| |
| out := string(buf) |
| c.Assert(out, checker.Contains, "from baz") |
| } |
| |
| func (s *DockerSuite) TestBuildAPIDoubleDockerfile(c *check.C) { |
| testRequires(c, UnixCli) // dockerfile overwrites Dockerfile on Windows |
| git := fakegit.New(c, "repo", map[string]string{ |
| "Dockerfile": `FROM busybox |
| RUN echo from Dockerfile`, |
| "dockerfile": `FROM busybox |
| RUN echo from dockerfile`, |
| }, false) |
| defer git.Close() |
| |
| // Make sure it tries to 'dockerfile' query param value |
| res, body, err := request.Post("/build?remote="+git.RepoURL, request.JSON) |
| c.Assert(err, checker.IsNil) |
| c.Assert(res.StatusCode, checker.Equals, http.StatusOK) |
| |
| buf, err := request.ReadBody(body) |
| c.Assert(err, checker.IsNil) |
| |
| out := string(buf) |
| c.Assert(out, checker.Contains, "from Dockerfile") |
| } |
| |
| func (s *DockerSuite) TestBuildAPIUnnormalizedTarPaths(c *check.C) { |
| // Make sure that build context tars with entries of the form |
| // x/./y don't cause caching false positives. |
| |
| buildFromTarContext := func(fileContents []byte) string { |
| buffer := new(bytes.Buffer) |
| tw := tar.NewWriter(buffer) |
| defer tw.Close() |
| |
| dockerfile := []byte(`FROM busybox |
| COPY dir /dir/`) |
| err := tw.WriteHeader(&tar.Header{ |
| Name: "Dockerfile", |
| Size: int64(len(dockerfile)), |
| }) |
| //failed to write tar file header |
| c.Assert(err, checker.IsNil) |
| |
| _, err = tw.Write(dockerfile) |
| // failed to write Dockerfile in tar file content |
| c.Assert(err, checker.IsNil) |
| |
| err = tw.WriteHeader(&tar.Header{ |
| Name: "dir/./file", |
| Size: int64(len(fileContents)), |
| }) |
| //failed to write tar file header |
| c.Assert(err, checker.IsNil) |
| |
| _, err = tw.Write(fileContents) |
| // failed to write file contents in tar file content |
| c.Assert(err, checker.IsNil) |
| |
| // failed to close tar archive |
| c.Assert(tw.Close(), checker.IsNil) |
| |
| res, body, err := request.Post("/build", request.RawContent(ioutil.NopCloser(buffer)), request.ContentType("application/x-tar")) |
| c.Assert(err, checker.IsNil) |
| c.Assert(res.StatusCode, checker.Equals, http.StatusOK) |
| |
| out, err := request.ReadBody(body) |
| c.Assert(err, checker.IsNil) |
| lines := strings.Split(string(out), "\n") |
| c.Assert(len(lines), checker.GreaterThan, 1) |
| c.Assert(lines[len(lines)-2], checker.Matches, ".*Successfully built [0-9a-f]{12}.*") |
| |
| re := regexp.MustCompile("Successfully built ([0-9a-f]{12})") |
| matches := re.FindStringSubmatch(lines[len(lines)-2]) |
| return matches[1] |
| } |
| |
| imageA := buildFromTarContext([]byte("abc")) |
| imageB := buildFromTarContext([]byte("def")) |
| |
| c.Assert(imageA, checker.Not(checker.Equals), imageB) |
| } |
| |
| func (s *DockerSuite) TestBuildOnBuildWithCopy(c *check.C) { |
| dockerfile := ` |
| FROM ` + minimalBaseImage() + ` as onbuildbase |
| ONBUILD COPY file /file |
| |
| FROM onbuildbase |
| ` |
| ctx := fakecontext.New(c, "", |
| fakecontext.WithDockerfile(dockerfile), |
| fakecontext.WithFile("file", "some content"), |
| ) |
| defer ctx.Close() |
| |
| res, body, err := request.Post( |
| "/build", |
| request.RawContent(ctx.AsTarReader(c)), |
| request.ContentType("application/x-tar")) |
| c.Assert(err, checker.IsNil) |
| c.Assert(res.StatusCode, checker.Equals, http.StatusOK) |
| |
| out, err := request.ReadBody(body) |
| c.Assert(err, checker.IsNil) |
| c.Assert(string(out), checker.Contains, "Successfully built") |
| } |
| |
| func (s *DockerSuite) TestBuildOnBuildCache(c *check.C) { |
| build := func(dockerfile string) []byte { |
| ctx := fakecontext.New(c, "", |
| fakecontext.WithDockerfile(dockerfile), |
| ) |
| defer ctx.Close() |
| |
| res, body, err := request.Post( |
| "/build", |
| request.RawContent(ctx.AsTarReader(c)), |
| request.ContentType("application/x-tar")) |
| require.NoError(c, err) |
| assert.Equal(c, http.StatusOK, res.StatusCode) |
| |
| out, err := request.ReadBody(body) |
| require.NoError(c, err) |
| assert.Contains(c, string(out), "Successfully built") |
| return out |
| } |
| |
| dockerfile := ` |
| FROM ` + minimalBaseImage() + ` as onbuildbase |
| ENV something=bar |
| ONBUILD ENV foo=bar |
| ` |
| build(dockerfile) |
| |
| dockerfile += "FROM onbuildbase" |
| out := build(dockerfile) |
| |
| imageIDs := getImageIDsFromBuild(c, out) |
| assert.Len(c, imageIDs, 2) |
| parentID, childID := imageIDs[0], imageIDs[1] |
| |
| client, err := request.NewClient() |
| require.NoError(c, err) |
| |
| // check parentID is correct |
| image, _, err := client.ImageInspectWithRaw(context.Background(), childID) |
| require.NoError(c, err) |
| assert.Equal(c, parentID, image.Parent) |
| } |
| |
| func (s *DockerRegistrySuite) TestBuildCopyFromForcePull(c *check.C) { |
| client, err := request.NewClient() |
| require.NoError(c, err) |
| |
| repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) |
| // tag the image to upload it to the private registry |
| err = client.ImageTag(context.TODO(), "busybox", repoName) |
| assert.Nil(c, err) |
| // push the image to the registry |
| rc, err := client.ImagePush(context.TODO(), repoName, types.ImagePushOptions{RegistryAuth: "{}"}) |
| assert.Nil(c, err) |
| _, err = io.Copy(ioutil.Discard, rc) |
| assert.Nil(c, err) |
| |
| dockerfile := fmt.Sprintf(` |
| FROM %s AS foo |
| RUN touch abc |
| FROM %s |
| COPY --from=foo /abc / |
| `, repoName, repoName) |
| |
| ctx := fakecontext.New(c, "", |
| fakecontext.WithDockerfile(dockerfile), |
| ) |
| defer ctx.Close() |
| |
| res, body, err := request.Post( |
| "/build?pull=1", |
| request.RawContent(ctx.AsTarReader(c)), |
| request.ContentType("application/x-tar")) |
| require.NoError(c, err) |
| assert.Equal(c, http.StatusOK, res.StatusCode) |
| |
| out, err := request.ReadBody(body) |
| require.NoError(c, err) |
| assert.Contains(c, string(out), "Successfully built") |
| } |
| |
| func (s *DockerSuite) TestBuildAddRemoteNoDecompress(c *check.C) { |
| buffer := new(bytes.Buffer) |
| tw := tar.NewWriter(buffer) |
| dt := []byte("contents") |
| err := tw.WriteHeader(&tar.Header{ |
| Name: "foo", |
| Size: int64(len(dt)), |
| Mode: 0600, |
| Typeflag: tar.TypeReg, |
| }) |
| require.NoError(c, err) |
| _, err = tw.Write(dt) |
| require.NoError(c, err) |
| err = tw.Close() |
| require.NoError(c, err) |
| |
| server := fakestorage.New(c, "", fakecontext.WithBinaryFiles(map[string]*bytes.Buffer{ |
| "test.tar": buffer, |
| })) |
| defer server.Close() |
| |
| dockerfile := fmt.Sprintf(` |
| FROM busybox |
| ADD %s/test.tar / |
| RUN [ -f test.tar ] |
| `, server.URL()) |
| |
| ctx := fakecontext.New(c, "", |
| fakecontext.WithDockerfile(dockerfile), |
| ) |
| defer ctx.Close() |
| |
| res, body, err := request.Post( |
| "/build", |
| request.RawContent(ctx.AsTarReader(c)), |
| request.ContentType("application/x-tar")) |
| require.NoError(c, err) |
| assert.Equal(c, http.StatusOK, res.StatusCode) |
| |
| out, err := request.ReadBody(body) |
| require.NoError(c, err) |
| assert.Contains(c, string(out), "Successfully built") |
| } |
| |
| func (s *DockerSuite) TestBuildChownOnCopy(c *check.C) { |
| testRequires(c, DaemonIsLinux) |
| dockerfile := `FROM busybox |
| RUN echo 'test1:x:1001:1001::/bin:/bin/false' >> /etc/passwd |
| RUN echo 'test1:x:1001:' >> /etc/group |
| RUN echo 'test2:x:1002:' >> /etc/group |
| COPY --chown=test1:1002 . /new_dir |
| RUN ls -l / |
| RUN [ $(ls -l / | grep new_dir | awk '{print $3":"$4}') = 'test1:test2' ] |
| RUN [ $(ls -nl / | grep new_dir | awk '{print $3":"$4}') = '1001:1002' ] |
| ` |
| ctx := fakecontext.New(c, "", |
| fakecontext.WithDockerfile(dockerfile), |
| fakecontext.WithFile("test_file1", "some test content"), |
| ) |
| defer ctx.Close() |
| |
| res, body, err := request.Post( |
| "/build", |
| request.RawContent(ctx.AsTarReader(c)), |
| request.ContentType("application/x-tar")) |
| c.Assert(err, checker.IsNil) |
| c.Assert(res.StatusCode, checker.Equals, http.StatusOK) |
| |
| out, err := request.ReadBody(body) |
| require.NoError(c, err) |
| assert.Contains(c, string(out), "Successfully built") |
| } |
| |
| func (s *DockerSuite) TestBuildCopyCacheOnFileChange(c *check.C) { |
| |
| dockerfile := `FROM busybox |
| COPY file /file` |
| |
| ctx1 := fakecontext.New(c, "", |
| fakecontext.WithDockerfile(dockerfile), |
| fakecontext.WithFile("file", "foo")) |
| ctx2 := fakecontext.New(c, "", |
| fakecontext.WithDockerfile(dockerfile), |
| fakecontext.WithFile("file", "bar")) |
| |
| var build = func(ctx *fakecontext.Fake) string { |
| res, body, err := request.Post("/build", |
| request.RawContent(ctx.AsTarReader(c)), |
| request.ContentType("application/x-tar")) |
| |
| require.NoError(c, err) |
| assert.Equal(c, http.StatusOK, res.StatusCode) |
| |
| out, err := request.ReadBody(body) |
| |
| ids := getImageIDsFromBuild(c, out) |
| return ids[len(ids)-1] |
| } |
| |
| id1 := build(ctx1) |
| id2 := build(ctx1) |
| id3 := build(ctx2) |
| |
| if id1 != id2 { |
| c.Fatal("didn't use the cache") |
| } |
| if id1 == id3 { |
| c.Fatal("COPY With different source file should not share same cache") |
| } |
| } |
| |
| func (s *DockerSuite) TestBuildAddCacheOnFileChange(c *check.C) { |
| |
| dockerfile := `FROM busybox |
| ADD file /file` |
| |
| ctx1 := fakecontext.New(c, "", |
| fakecontext.WithDockerfile(dockerfile), |
| fakecontext.WithFile("file", "foo")) |
| ctx2 := fakecontext.New(c, "", |
| fakecontext.WithDockerfile(dockerfile), |
| fakecontext.WithFile("file", "bar")) |
| |
| var build = func(ctx *fakecontext.Fake) string { |
| res, body, err := request.Post("/build", |
| request.RawContent(ctx.AsTarReader(c)), |
| request.ContentType("application/x-tar")) |
| |
| require.NoError(c, err) |
| assert.Equal(c, http.StatusOK, res.StatusCode) |
| |
| out, err := request.ReadBody(body) |
| |
| ids := getImageIDsFromBuild(c, out) |
| return ids[len(ids)-1] |
| } |
| |
| id1 := build(ctx1) |
| id2 := build(ctx1) |
| id3 := build(ctx2) |
| |
| if id1 != id2 { |
| c.Fatal("didn't use the cache") |
| } |
| if id1 == id3 { |
| c.Fatal("COPY With different source file should not share same cache") |
| } |
| } |
| |
| func (s *DockerSuite) TestBuildWithSession(c *check.C) { |
| testRequires(c, ExperimentalDaemon) |
| |
| dockerfile := ` |
| FROM busybox |
| COPY file / |
| RUN cat /file |
| ` |
| |
| fctx := fakecontext.New(c, "", |
| fakecontext.WithFile("file", "some content"), |
| ) |
| defer fctx.Close() |
| |
| out := testBuildWithSession(c, fctx.Dir, dockerfile) |
| assert.Contains(c, out, "some content") |
| |
| fctx.Add("second", "contentcontent") |
| |
| dockerfile += ` |
| COPY second / |
| RUN cat /second |
| ` |
| |
| out = testBuildWithSession(c, fctx.Dir, dockerfile) |
| assert.Equal(c, strings.Count(out, "Using cache"), 2) |
| assert.Contains(c, out, "contentcontent") |
| |
| client, err := request.NewClient() |
| require.NoError(c, err) |
| |
| du, err := client.DiskUsage(context.TODO()) |
| assert.Nil(c, err) |
| assert.True(c, du.BuilderSize > 10) |
| |
| out = testBuildWithSession(c, fctx.Dir, dockerfile) |
| assert.Equal(c, strings.Count(out, "Using cache"), 4) |
| |
| du2, err := client.DiskUsage(context.TODO()) |
| assert.Nil(c, err) |
| assert.Equal(c, du.BuilderSize, du2.BuilderSize) |
| |
| // rebuild with regular tar, confirm cache still applies |
| fctx.Add("Dockerfile", dockerfile) |
| res, body, err := request.Post( |
| "/build", |
| request.RawContent(fctx.AsTarReader(c)), |
| request.ContentType("application/x-tar")) |
| require.NoError(c, err) |
| assert.Equal(c, http.StatusOK, res.StatusCode) |
| |
| outBytes, err := request.ReadBody(body) |
| require.NoError(c, err) |
| assert.Contains(c, string(outBytes), "Successfully built") |
| assert.Equal(c, strings.Count(string(outBytes), "Using cache"), 4) |
| |
| _, err = client.BuildCachePrune(context.TODO()) |
| assert.Nil(c, err) |
| |
| du, err = client.DiskUsage(context.TODO()) |
| assert.Nil(c, err) |
| assert.Equal(c, du.BuilderSize, int64(0)) |
| } |
| |
| func testBuildWithSession(c *check.C, dir, dockerfile string) (outStr string) { |
| client, err := request.NewClient() |
| require.NoError(c, err) |
| |
| sess, err := session.NewSession("foo1", "foo") |
| assert.Nil(c, err) |
| |
| fsProvider := filesync.NewFSSyncProvider([]filesync.SyncedDir{ |
| {Dir: dir}, |
| }) |
| sess.Allow(fsProvider) |
| |
| g, ctx := errgroup.WithContext(context.Background()) |
| |
| g.Go(func() error { |
| return sess.Run(ctx, client.DialSession) |
| }) |
| |
| g.Go(func() error { |
| res, body, err := request.Post("/build?remote=client-session&session="+sess.ID(), func(req *http.Request) error { |
| req.Body = ioutil.NopCloser(strings.NewReader(dockerfile)) |
| return nil |
| }) |
| if err != nil { |
| return err |
| } |
| assert.Equal(c, res.StatusCode, http.StatusOK) |
| out, err := request.ReadBody(body) |
| require.NoError(c, err) |
| assert.Contains(c, string(out), "Successfully built") |
| sess.Close() |
| outStr = string(out) |
| return nil |
| }) |
| |
| err = g.Wait() |
| assert.Nil(c, err) |
| return |
| } |
| |
| type buildLine struct { |
| Stream string |
| Aux struct { |
| ID string |
| } |
| } |
| |
| func getImageIDsFromBuild(c *check.C, output []byte) []string { |
| ids := []string{} |
| for _, line := range bytes.Split(output, []byte("\n")) { |
| if len(line) == 0 { |
| continue |
| } |
| entry := buildLine{} |
| require.NoError(c, json.Unmarshal(line, &entry)) |
| if entry.Aux.ID != "" { |
| ids = append(ids, entry.Aux.ID) |
| } |
| } |
| return ids |
| } |