ci(deploy): move the images' :latest only when a release is promoted (#1716)
This commit is contained in:
8 files changed
+100
-46
No files matched your search
@@ -36,7 +36,7 @@ jobs:
|
|||||||
- name: Build archied
|
- name: Build archied
|
||||||
run: go build -o "$RUNNER_TEMP/archied" ./cmd/archied
|
run: go build -o "$RUNNER_TEMP/archied" ./cmd/archied
|
||||||
|
|
||||||
# Promotion retags stable in the registry.
|
# Promotion retags stable and the images' latest in the registry.
|
||||||
- name: Log in to the Gitea registry
|
- name: Log in to the Gitea registry
|
||||||
run: echo "${{ secrets.DEPLOY_TOKEN }}" | docker login git.catlow.cloud -u "${{ github.actor }}" --password-stdin
|
run: echo "${{ secrets.DEPLOY_TOKEN }}" | docker login git.catlow.cloud -u "${{ github.actor }}" --password-stdin
|
||||||
|
|
||||||
@@ -44,4 +44,5 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
ARCHIE_CANARY_TOKEN: ${{ secrets.CANARY_TOKEN }}
|
ARCHIE_CANARY_TOKEN: ${{ secrets.CANARY_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
"$RUNNER_TEMP/archied" release canary check -dashboard "${{ vars.CANARY_URL }}"
|
"$RUNNER_TEMP/archied" release canary check -dashboard "${{ vars.CANARY_URL }}" \
|
||||||
|
-latest git.catlow.cloud/sam/archied,git.catlow.cloud/sam/archie-agent
|
||||||
@@ -138,7 +138,7 @@ jobs:
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
if ! git tag --points-at HEAD | grep -qE '^v[0-9]' ; then
|
if ! git tag --points-at HEAD | grep -qE '^v[0-9]' ; then
|
||||||
echo "This commit is not a release, so the images are published as :edge and sha-<sha>; :latest keeps naming the newest release."
|
echo "This commit is not a release, so the images are published as :edge and sha-<sha>; :latest keeps naming the newest promoted release."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
archied-image:
|
archied-image:
|
||||||
@@ -185,19 +185,18 @@ jobs:
|
|||||||
echo "release_tag=$RELEASE_TAG" >> "$GITHUB_OUTPUT"
|
echo "release_tag=$RELEASE_TAG" >> "$GITHUB_OUTPUT"
|
||||||
echo "gateway=$GATEWAY_VERSION" >> "$GITHUB_OUTPUT"
|
echo "gateway=$GATEWAY_VERSION" >> "$GITHUB_OUTPUT"
|
||||||
echo "runtime=$RUNTIME_VERSION" >> "$GITHUB_OUTPUT"
|
echo "runtime=$RUNTIME_VERSION" >> "$GITHUB_OUTPUT"
|
||||||
# :latest names the newest RELEASE, so only a commit that IS one
|
# :latest names the newest PROMOTED release, so a release publishes
|
||||||
# publishes it. A push to main publishes :edge (mutable, newest main)
|
# :v<version> and canary promotion moves :latest to it. A push to
|
||||||
# and sha-<sha> (immutable) instead. That is what stops a release's
|
# main publishes :edge (mutable, newest main) and sha-<sha>
|
||||||
# :latest stamping from being overwritten minutes later by the next
|
# (immutable), so an ordinary commit never overwrites a release's
|
||||||
# ordinary commit, which left a host reporting dev and made the update
|
# stamping.
|
||||||
# path unable to tell current from stale (archie-core-i5nb).
|
|
||||||
#
|
#
|
||||||
# `if` blocks rather than `[ -n "$X" ] && echo`: the latter returns
|
# `if` blocks rather than `[ -n "$X" ] && echo`: the latter returns
|
||||||
# non-zero when the test fails, and these steps run under `set -e`.
|
# non-zero when the test fails, and these steps run under `set -e`.
|
||||||
{
|
{
|
||||||
echo "archied_tags<<EOF"
|
echo "archied_tags<<EOF"
|
||||||
if [ -n "$RELEASE_TAG" ]; then
|
if [ -n "$RELEASE_TAG" ]; then
|
||||||
echo "git.catlow.cloud/sam/archied:latest"
|
echo "git.catlow.cloud/sam/archied:$RELEASE_TAG"
|
||||||
fi
|
fi
|
||||||
echo "git.catlow.cloud/sam/archied:edge"
|
echo "git.catlow.cloud/sam/archied:edge"
|
||||||
echo "git.catlow.cloud/sam/archied:sha-${{ steps.commit.outputs.sha }}"
|
echo "git.catlow.cloud/sam/archied:sha-${{ steps.commit.outputs.sha }}"
|
||||||
@@ -260,7 +259,7 @@ jobs:
|
|||||||
file: Dockerfile
|
file: Dockerfile
|
||||||
push: true
|
push: true
|
||||||
tags: |
|
tags: |
|
||||||
git.catlow.cloud/sam/archie-agent:latest
|
git.catlow.cloud/sam/archie-agent:${{ steps.versions.outputs.release_tag }}
|
||||||
git.catlow.cloud/sam/archie-agent:sha-${{ steps.commit.outputs.sha }}
|
git.catlow.cloud/sam/archie-agent:sha-${{ steps.commit.outputs.sha }}
|
||||||
build-args: |
|
build-args: |
|
||||||
RUNTIME_VERSION=${{ steps.versions.outputs.runtime }}
|
RUNTIME_VERSION=${{ steps.versions.outputs.runtime }}
|
||||||
|
|||||||
+1
-1
@@ -7,7 +7,7 @@ release's per-component sections are labelled beneath its heading.
|
|||||||
|
|
||||||
### archied
|
### archied
|
||||||
|
|
||||||
- A release reaches `stable` only after it has run on a canary for a day and passed: `archied release publish` now publishes to `next` alone, `archied release canary start|check` installs a release on a canary and judges it from the canary's dashboard, promoting it or rolling the canary back, and `archied release promote` moves `stable` by hand. Promotion refuses a prerelease and a release older than the one `stable` names.
|
- A release reaches `stable` only after it has run on a canary for a day and passed: `archied release publish` now publishes to `next` alone, `archied release canary start|check` installs a release on a canary and judges it from the canary's dashboard, promoting it or rolling the canary back, and `archied release promote` moves `stable` by hand. Promotion refuses a prerelease and a release older than the one `stable` names. The `archied` and `archie-agent` images are published as `:v<version>`, and `:latest` moves only when a release is promoted.
|
||||||
|
|
||||||
## [1.56.0] - 2026-10-31
|
## [1.56.0] - 2026-10-31
|
||||||
|
|
||||||
|
|||||||
+8
-5
@@ -57,9 +57,9 @@ The tag push runs `deploy.yml`:
|
|||||||
- **Gate.** `task check TEST_FLAGS=-count=1` (uncached, including the
|
- **Gate.** `task check TEST_FLAGS=-count=1` (uncached, including the
|
||||||
Docker-backed tests `-short` skips) and `govulncheck`, then a check that the
|
Docker-backed tests `-short` skips) and `govulncheck`, then a check that the
|
||||||
tag has a `[version]` section in `CHANGELOG.md`.
|
tag has a `[version]` section in `CHANGELOG.md`.
|
||||||
- **Images.** `archied:latest` and `archie-agent:latest` on a release tag;
|
- **Images.** `archied:v<version>` and `archie-agent:v<version>` on a release
|
||||||
`archied:edge` and `archied:sha-<commit>` on any other push to `main`, so
|
tag; `archied:edge` and `archied:sha-<commit>` on any other push to `main`.
|
||||||
`:latest` always names the newest release.
|
`:latest` moves only on promotion, so it names the newest promoted release.
|
||||||
- **The signed release.** `archied release publish` signs a statement
|
- **The signed release.** `archied release publish` signs a statement
|
||||||
(version, issued_at, zip digest, agent image by digest) with
|
(version, issued_at, zip digest, agent image by digest) with
|
||||||
`RELEASE_SIGNING_KEY` and pushes it with the distribution zip to
|
`RELEASE_SIGNING_KEY` and pushes it with the distribution zip to
|
||||||
@@ -104,7 +104,9 @@ through its dashboard API and judges the release on `next`:
|
|||||||
|
|
||||||
A release that passes the window is promoted: `stable` is retagged to the
|
A release that passes the window is promoted: `stable` is retagged to the
|
||||||
`v<version>` manifest. The signature covers the statement, not the tag, so the
|
`v<version>` manifest. The signature covers the statement, not the tag, so the
|
||||||
retag needs no signing key. Promotion refuses a prerelease and a release issued
|
retag needs no signing key. The `archied` and `archie-agent` images' `:latest`
|
||||||
|
move to their `:v<version>` first, and the agent image must be the digest the
|
||||||
|
statement pins. Promotion refuses a prerelease and a release issued
|
||||||
before the one `stable` names, the same replay rule hosts enforce. The decision
|
before the one `stable` names, the same replay rule hosts enforce. The decision
|
||||||
and its reasons are in the `canary` run log.
|
and its reasons are in the `canary` run log.
|
||||||
|
|
||||||
@@ -134,5 +136,6 @@ The runner must reach `CANARY_URL`. Without a canary a release stays on `next`
|
|||||||
until promoted by hand, under the same refusals:
|
until promoted by hand, under the same refusals:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
archied release promote -version 1.57.0
|
archied release promote -version 1.57.0 \
|
||||||
|
-latest git.catlow.cloud/sam/archied,git.catlow.cloud/sam/archie-agent
|
||||||
```
|
```
|
||||||
@@ -124,6 +124,7 @@ func runReleasePromote(args []string, out, stderr io.Writer) int {
|
|||||||
reference := flags.String("source", "", "OCI repository releases are published to; blank is the official releases")
|
reference := flags.String("source", "", "OCI repository releases are published to; blank is the official releases")
|
||||||
key := flags.String("key", "", "base64 ed25519 public key the source signs with")
|
key := flags.String("key", "", "base64 ed25519 public key the source signs with")
|
||||||
version := flags.String("version", "", "release to promote")
|
version := flags.String("version", "", "release to promote")
|
||||||
|
latest := flags.String("latest", "", "comma-separated image repositories whose v<version> tag becomes latest")
|
||||||
if err := flags.Parse(args); err != nil {
|
if err := flags.Parse(args); err != nil {
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
@@ -132,7 +133,7 @@ func runReleasePromote(args []string, out, stderr io.Writer) int {
|
|||||||
fmt.Fprintln(stderr, "release promote: -version is required, and -source with -key name another source")
|
fmt.Fprintln(stderr, "release promote: -version is required, and -source with -key name another source")
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
statement, err := releaseupdate.Promote(context.Background(), source.Reference, source.Key, *version)
|
statement, err := releaseupdate.Promote(context.Background(), source.Reference, source.Key, *version, splitList(*latest))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(stderr, err)
|
fmt.Fprintln(stderr, err)
|
||||||
return 1
|
return 1
|
||||||
@@ -140,3 +141,13 @@ func runReleasePromote(args []string, out, stderr io.Writer) int {
|
|||||||
fmt.Fprintf(out, "stable is %s\n", statement.Version)
|
fmt.Fprintf(out, "stable is %s\n", statement.Version)
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func splitList(value string) []string {
|
||||||
|
var items []string
|
||||||
|
for item := range strings.SplitSeq(value, ",") {
|
||||||
|
if item = strings.TrimSpace(item); item != "" {
|
||||||
|
items = append(items, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return items
|
||||||
|
}
|
||||||
@@ -36,6 +36,7 @@ func runReleaseCanary(args []string, out, stderr io.Writer) int {
|
|||||||
version := flags.String("version", "", "release to install (start)")
|
version := flags.String("version", "", "release to install (start)")
|
||||||
reference := flags.String("source", "", "OCI repository releases are published to; blank is the official releases (check)")
|
reference := flags.String("source", "", "OCI repository releases are published to; blank is the official releases (check)")
|
||||||
key := flags.String("key", "", "base64 ed25519 public key the source signs with (check)")
|
key := flags.String("key", "", "base64 ed25519 public key the source signs with (check)")
|
||||||
|
latest := flags.String("latest", "", "comma-separated image repositories whose v<version> tag becomes latest on promotion (check)")
|
||||||
checkout := flags.String("checkout", ".", "repository checkout with release tags, to see whether a release migrated the database (check)")
|
checkout := flags.String("checkout", ".", "repository checkout with release tags, to see whether a release migrated the database (check)")
|
||||||
if err := flags.Parse(args[1:]); err != nil {
|
if err := flags.Parse(args[1:]); err != nil {
|
||||||
return 2
|
return 2
|
||||||
@@ -63,7 +64,7 @@ func runReleaseCanary(args []string, out, stderr io.Writer) int {
|
|||||||
fmt.Fprintln(stderr, "release canary check:", err)
|
fmt.Fprintln(stderr, "release canary check:", err)
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
rolledBack, err := checkCanary(ctx, dashboard, source, *checkout, out)
|
rolledBack, err := checkCanary(ctx, dashboard, source, splitList(*latest), *checkout, out)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(stderr, err)
|
fmt.Fprintln(stderr, err)
|
||||||
return 1
|
return 1
|
||||||
@@ -83,7 +84,7 @@ func startCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, v
|
|||||||
|
|
||||||
// checkCanary judges the release on next once and acts on the verdict. It
|
// checkCanary judges the release on next once and acts on the verdict. It
|
||||||
// reports rolledBack so a failed canary turns the run red.
|
// reports rolledBack so a failed canary turns the run red.
|
||||||
func checkCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, checkout string, out io.Writer) (rolledBack bool, err error) {
|
func checkCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, source distribution.Source, latest []string, checkout string, out io.Writer) (rolledBack bool, err error) {
|
||||||
next, err := releaseupdate.Fetch(ctx, source, "next")
|
next, err := releaseupdate.Fetch(ctx, source, "next")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
@@ -118,7 +119,7 @@ func checkCanary(ctx context.Context, dashboard releaseupdate.CanaryDashboard, s
|
|||||||
}
|
}
|
||||||
switch verdict.Decision {
|
switch verdict.Decision {
|
||||||
case releaseupdate.CanaryPromote:
|
case releaseupdate.CanaryPromote:
|
||||||
if _, err := releaseupdate.Promote(ctx, source.Reference, source.Key, release.Version); err != nil {
|
if _, err := releaseupdate.Promote(ctx, source.Reference, source.Key, release.Version, latest); err != nil {
|
||||||
return false, fmt.Errorf("promote %s: %w", release.Version, err)
|
return false, fmt.Errorf("promote %s: %w", release.Version, err)
|
||||||
}
|
}
|
||||||
fmt.Fprintf(out, "stable is %s\n", release.Version)
|
fmt.Fprintf(out, "stable is %s\n", release.Version)
|
||||||
|
|||||||
@@ -7,10 +7,12 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"strings"
|
||||||
|
|
||||||
digest "github.com/opencontainers/go-digest"
|
digest "github.com/opencontainers/go-digest"
|
||||||
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||||
"oras.land/oras-go/v2/errdef"
|
"oras.land/oras-go/v2/errdef"
|
||||||
|
"oras.land/oras-go/v2/registry/remote"
|
||||||
|
|
||||||
"github.com/samcharles93/archie-core/internal/domain/distribution"
|
"github.com/samcharles93/archie-core/internal/domain/distribution"
|
||||||
"github.com/samcharles93/archie-core/internal/infrastructure/signedoci"
|
"github.com/samcharles93/archie-core/internal/infrastructure/signedoci"
|
||||||
@@ -20,7 +22,10 @@ import (
|
|||||||
// covers the statement, not the tag, so stable is the same signed manifest
|
// covers the statement, not the tag, so stable is the same signed manifest
|
||||||
// and needs no re-signing. A prerelease, or a release issued before the one
|
// and needs no re-signing. A prerelease, or a release issued before the one
|
||||||
// stable names, is refused: hosts would refuse the latter as a replay.
|
// stable names, is refused: hosts would refuse the latter as a replay.
|
||||||
func Promote(ctx context.Context, repository string, key ed25519.PublicKey, version string) (Statement, error) {
|
//
|
||||||
|
// Each image repository in latest has its v<version> tag copied to latest
|
||||||
|
// first, so a failure leaves stable unmoved and a rerun finishes the job.
|
||||||
|
func Promote(ctx context.Context, repository string, key ed25519.PublicKey, version string, latest []string) (Statement, error) {
|
||||||
source := distribution.Source{Reference: repository, Key: key}
|
source := distribution.Source{Reference: repository, Key: key}
|
||||||
release, err := Fetch(ctx, source, Tag("exact-pin", version))
|
release, err := Fetch(ctx, source, Tag("exact-pin", version))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -41,34 +46,68 @@ func Promote(ctx context.Context, repository string, key ed25519.PublicKey, vers
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return Statement{}, err
|
return Statement{}, err
|
||||||
}
|
}
|
||||||
// Tag the manifest the version tag resolves to now, after checking it
|
desc, err := verifiedManifest(ctx, repo, version, release.Artifact.Statement, key)
|
||||||
// carries the statement just verified, so a tag moved in between cannot
|
|
||||||
// slip an unverified manifest onto stable.
|
|
||||||
desc, err := repo.Resolve(ctx, Tag("exact-pin", version))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Statement{}, fmt.Errorf("resolve %s: %w", version, err)
|
return Statement{}, err
|
||||||
}
|
}
|
||||||
stream, err := repo.Fetch(ctx, desc)
|
for _, image := range latest {
|
||||||
if err != nil {
|
if err := promoteImage(ctx, image, version, release.Statement.AgentImage); err != nil {
|
||||||
return Statement{}, fmt.Errorf("fetch %s manifest: %w", version, err)
|
return Statement{}, err
|
||||||
}
|
}
|
||||||
raw, err := io.ReadAll(io.LimitReader(stream, desc.Size))
|
|
||||||
_ = stream.Close()
|
|
||||||
if err != nil {
|
|
||||||
return Statement{}, fmt.Errorf("read %s manifest: %w", version, err)
|
|
||||||
}
|
|
||||||
if digest.FromBytes(raw) != desc.Digest {
|
|
||||||
return Statement{}, fmt.Errorf("%s manifest does not match its digest", version)
|
|
||||||
}
|
|
||||||
var manifest ocispec.Manifest
|
|
||||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
|
||||||
return Statement{}, fmt.Errorf("%s manifest: %w", version, err)
|
|
||||||
}
|
|
||||||
if err := signedoci.Verify(manifest, release.Artifact.Statement, key, Annotation); err != nil {
|
|
||||||
return Statement{}, fmt.Errorf("%s changed while promoting: %w", version, err)
|
|
||||||
}
|
}
|
||||||
if err := repo.Tag(ctx, desc, "stable"); err != nil {
|
if err := repo.Tag(ctx, desc, "stable"); err != nil {
|
||||||
return Statement{}, fmt.Errorf("tag stable: %w", err)
|
return Statement{}, fmt.Errorf("tag stable: %w", err)
|
||||||
}
|
}
|
||||||
return release.Statement, nil
|
return release.Statement, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// promoteImage tags image:v<version> as latest. The agent image must be the
|
||||||
|
// one the statement pins, or latest would name an image the release never
|
||||||
|
// signed for.
|
||||||
|
func promoteImage(ctx context.Context, image, version, agentImage string) error {
|
||||||
|
repo, err := pushRepository(image)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
desc, err := repo.Resolve(ctx, Tag("exact-pin", version))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("resolve %s:v%s: %w", image, version, err)
|
||||||
|
}
|
||||||
|
if pinned, digest, ok := strings.Cut(agentImage, "@"); ok && pinned == image && desc.Digest.String() != digest {
|
||||||
|
return fmt.Errorf("%s:v%s is %s, not the %s the release pins", image, version, desc.Digest, digest)
|
||||||
|
}
|
||||||
|
if err := repo.Tag(ctx, desc, "latest"); err != nil {
|
||||||
|
return fmt.Errorf("tag %s:latest: %w", image, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// verifiedManifest is the manifest the version tag resolves to now, checked to
|
||||||
|
// carry the statement already verified, so a tag moved in between cannot slip
|
||||||
|
// an unverified manifest onto stable.
|
||||||
|
func verifiedManifest(ctx context.Context, repo *remote.Repository, version string, statement []byte, key ed25519.PublicKey) (ocispec.Descriptor, error) {
|
||||||
|
desc, err := repo.Resolve(ctx, Tag("exact-pin", version))
|
||||||
|
if err != nil {
|
||||||
|
return ocispec.Descriptor{}, fmt.Errorf("resolve %s: %w", version, err)
|
||||||
|
}
|
||||||
|
stream, err := repo.Fetch(ctx, desc)
|
||||||
|
if err != nil {
|
||||||
|
return ocispec.Descriptor{}, fmt.Errorf("fetch %s manifest: %w", version, err)
|
||||||
|
}
|
||||||
|
raw, err := io.ReadAll(io.LimitReader(stream, desc.Size))
|
||||||
|
_ = stream.Close()
|
||||||
|
if err != nil {
|
||||||
|
return ocispec.Descriptor{}, fmt.Errorf("read %s manifest: %w", version, err)
|
||||||
|
}
|
||||||
|
if digest.FromBytes(raw) != desc.Digest {
|
||||||
|
return ocispec.Descriptor{}, fmt.Errorf("%s manifest does not match its digest", version)
|
||||||
|
}
|
||||||
|
var manifest ocispec.Manifest
|
||||||
|
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||||
|
return ocispec.Descriptor{}, fmt.Errorf("%s manifest: %w", version, err)
|
||||||
|
}
|
||||||
|
if err := signedoci.Verify(manifest, statement, key, Annotation); err != nil {
|
||||||
|
return ocispec.Descriptor{}, fmt.Errorf("%s changed while promoting: %w", version, err)
|
||||||
|
}
|
||||||
|
return desc, nil
|
||||||
|
}
|
||||||
@@ -56,7 +56,7 @@ func TestPromote(t *testing.T) {
|
|||||||
registry.publish(t, r.version, r.issued, r.prerelease, r.signer, r.tags...)
|
registry.publish(t, r.version, r.issued, r.prerelease, r.signer, r.tags...)
|
||||||
}
|
}
|
||||||
repository := strings.TrimPrefix(server.URL, "http://") + "/archie"
|
repository := strings.TrimPrefix(server.URL, "http://") + "/archie"
|
||||||
_, err := Promote(t.Context(), repository, trusted, tc.promote)
|
_, err := Promote(t.Context(), repository, trusted, tc.promote, nil)
|
||||||
switch {
|
switch {
|
||||||
case tc.want == "" && err != nil:
|
case tc.want == "" && err != nil:
|
||||||
t.Fatalf("Promote: %v", err)
|
t.Fatalf("Promote: %v", err)
|
||||||
|
|||||||
Reference in new issue
Block a user