fix(workflow): one baseline fix per repository revision, drop download noise from gate output (#1806)
This commit is contained in:
12 files changed
+386
-13
No files matched your search
@@ -33,6 +33,7 @@ import (
|
||||
// worktreerpc; local git operations run on the bind-mounted worktree.
|
||||
type remoteTrees interface {
|
||||
Push(ctx context.Context) error
|
||||
workflow.BaselineLedger
|
||||
}
|
||||
|
||||
type hybridTrees struct {
|
||||
@@ -263,6 +264,9 @@ func runTask(ctx context.Context, req taskrun.Request, dependencies taskDependen
|
||||
Log: log,
|
||||
Guardrails: dependencies.guardrails,
|
||||
}
|
||||
if dependencies.trees != nil {
|
||||
tc.Baseline = dependencies.trees
|
||||
}
|
||||
tc.Instructions = req.Instructions
|
||||
configureTaskContext(tc, workDir)
|
||||
|
||||
|
||||
@@ -362,7 +362,7 @@ func registerTaskRPCServers(nc *natsio.Conn, resolve func(context.Context, *work
|
||||
resolved, err := resolve(ctx, t, config.OperationPR)
|
||||
return resolved.Forge, err
|
||||
}}
|
||||
trees := &worktreerpc.Server{Runs: runs, Log: log, Resolve: func(ctx context.Context, t *workflow.Task) (*worktree.Manager, error) {
|
||||
trees := &worktreerpc.Server{Runs: runs, Log: log, Baselines: worktreerpc.NewBaselineClaims(), Resolve: func(ctx context.Context, t *workflow.Task) (*worktree.Manager, error) {
|
||||
resolved, err := resolve(ctx, t, config.OperationPush)
|
||||
return resolved.Trees, err
|
||||
}}
|
||||
|
||||
@@ -35,6 +35,7 @@ func (b taskBroker) TaskSubjects(ctx context.Context, taskID int64, credential s
|
||||
agentexec.SubjectForEvents(taskID),
|
||||
spanrelay.Subject(taskID),
|
||||
worktreerpc.SubjectFor(task.Identity, worktreerpc.SubjectPush),
|
||||
worktreerpc.SubjectFor(task.Identity, worktreerpc.SubjectBaseline),
|
||||
messagerpc.SubjectFor(task.Identity),
|
||||
}, forgerpc.Subjects(task.Identity)...)
|
||||
return nats.TaskSubjects{Publish: publish, Subscribe: []string{agentnats.SubjectForTask(taskID)}}, nil
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// BaselineClaim answers a task's request to repair a red baseline.
|
||||
type BaselineClaim struct {
|
||||
// Lead is true when the caller owns the repair and must run it.
|
||||
Lead bool
|
||||
// Task is the task that owns the repair when Lead is false.
|
||||
Task int64
|
||||
// Done is true once that task has finished; Failure is its failure, empty
|
||||
// when it repaired the baseline on its own branch.
|
||||
Done bool
|
||||
Failure string
|
||||
}
|
||||
|
||||
// BaselineLedger makes one task repair a red baseline per repository revision
|
||||
// and command. Every task runs in its own container, so the ledger lives with
|
||||
// the daemon.
|
||||
type BaselineLedger interface {
|
||||
ClaimBaselineFix(ctx context.Context, key string) (BaselineClaim, error)
|
||||
FinishBaselineFix(ctx context.Context, key, failure string) error
|
||||
}
|
||||
|
||||
const defaultBaselinePoll = 15 * time.Second
|
||||
|
||||
// repairBaselineOnce runs the repair when this task leads it, or reports the
|
||||
// verdict of the task that does. Without a ledger every task repairs its own.
|
||||
func repairBaselineOnce(ctx context.Context, tc *TaskContext, argv []string, out []byte) error {
|
||||
if tc.Baseline == nil {
|
||||
return runBaselineFix(ctx, tc, argv, out)
|
||||
}
|
||||
revision, err := exec.CommandContext(ctx, "git", "-C", tc.Dir, "rev-parse", "HEAD").Output()
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve baseline revision: %w", err)
|
||||
}
|
||||
key := strings.TrimSpace(string(revision)) + " " + strings.Join(argv, " ")
|
||||
poll := tc.baselinePoll
|
||||
if poll <= 0 {
|
||||
poll = defaultBaselinePoll
|
||||
}
|
||||
for {
|
||||
claim, err := tc.Baseline.ClaimBaselineFix(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("claim baseline fix: %w", err)
|
||||
}
|
||||
switch {
|
||||
case claim.Lead:
|
||||
fixErr := runBaselineFix(ctx, tc, argv, out)
|
||||
failure := ""
|
||||
if fixErr != nil {
|
||||
failure = clip(fixErr.Error(), baselineParkOutputBytes)
|
||||
}
|
||||
if err := tc.Baseline.FinishBaselineFix(context.WithoutCancel(ctx), key, failure); err != nil {
|
||||
tc.Log.WarnContext(ctx, "release baseline fix claim", "err", err)
|
||||
}
|
||||
return fixErr
|
||||
case !claim.Done:
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(poll):
|
||||
}
|
||||
default:
|
||||
return baselineSharedError(argv, string(out), claim)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// gateProgressLine matches the package-download chatter `go` prints before the
|
||||
// real failure.
|
||||
var gateProgressLine = regexp.MustCompile(`^go: (downloading|finding|extracting) `)
|
||||
|
||||
// trimGateProgress drops download progress from gate output so the failing
|
||||
// command's own lines lead.
|
||||
func trimGateProgress(out string) string {
|
||||
if !strings.Contains(out, "go: ") {
|
||||
return out
|
||||
}
|
||||
lines := strings.Split(out, "\n")
|
||||
kept := lines[:0]
|
||||
for _, line := range lines {
|
||||
if !gateProgressLine.MatchString(line) {
|
||||
kept = append(kept, line)
|
||||
}
|
||||
}
|
||||
return strings.Join(kept, "\n")
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/agentrun"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
)
|
||||
|
||||
type countingRunner struct{ runs int }
|
||||
|
||||
func (r *countingRunner) Run(context.Context, string, agentrun.Request, agentrun.Reporter) (agentrun.Result, error) {
|
||||
r.runs++
|
||||
return agentrun.Result{Version: agentrun.ProtocolVersion, Status: "failed"}, nil
|
||||
}
|
||||
|
||||
// scriptedLedger answers claims from a fixed list, then repeats the last one.
|
||||
type scriptedLedger struct {
|
||||
claims []BaselineClaim
|
||||
finished []string
|
||||
}
|
||||
|
||||
func (l *scriptedLedger) ClaimBaselineFix(context.Context, string) (BaselineClaim, error) {
|
||||
claim := l.claims[0]
|
||||
if len(l.claims) > 1 {
|
||||
l.claims = l.claims[1:]
|
||||
}
|
||||
return claim, nil
|
||||
}
|
||||
|
||||
func (l *scriptedLedger) FinishBaselineFix(_ context.Context, _, failure string) error {
|
||||
l.finished = append(l.finished, failure)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestRedBaselineIsRepairedByOneTask(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
claims []BaselineClaim
|
||||
wantRuns int
|
||||
wantFinished int
|
||||
wantInErr string
|
||||
}{
|
||||
{
|
||||
name: "the task that leads repairs it",
|
||||
claims: []BaselineClaim{{Lead: true}},
|
||||
wantRuns: 1,
|
||||
wantFinished: 1,
|
||||
wantInErr: "did not repair it",
|
||||
},
|
||||
{
|
||||
name: "a task waits on the lead, then reports its failure",
|
||||
claims: []BaselineClaim{{Task: 7}, {Task: 7}, {Task: 7, Done: true, Failure: "builder status failed"}},
|
||||
wantInErr: "task 7 could not repair it: builder status failed",
|
||||
},
|
||||
{
|
||||
name: "a task reuses a repair made on another branch",
|
||||
claims: []BaselineClaim{{Task: 7, Done: true}},
|
||||
wantInErr: "task 7 repaired it on its own branch",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for _, args := range [][]string{{"init", "-q"}, {"-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", "base"}} {
|
||||
if out, err := exec.CommandContext(t.Context(), "git", append([]string{"-C", dir}, args...)...).CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %v\n%s", args, err, out)
|
||||
}
|
||||
}
|
||||
agent := &countingRunner{}
|
||||
ledger := &scriptedLedger{claims: tt.claims}
|
||||
tc := &TaskContext{
|
||||
Task: &task.Task{ID: 1, Attempt: 1, Source: task.SourceTest},
|
||||
Repo: config.Repo{Owner: "acme", Name: "widgets"},
|
||||
Cfg: config.Config{Models: map[string]string{"default": "m"}},
|
||||
Dir: dir,
|
||||
Log: slog.Default(),
|
||||
Agent: agent,
|
||||
Baseline: ledger,
|
||||
baselinePoll: time.Millisecond,
|
||||
}
|
||||
|
||||
err := repairBaselineOnce(context.Background(), tc, []string{"task", "check"}, []byte("FAIL"))
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantInErr) {
|
||||
t.Fatalf("err = %v, want it to contain %q", err, tt.wantInErr)
|
||||
}
|
||||
if agent.runs != tt.wantRuns {
|
||||
t.Errorf("fix agents run = %d, want %d", agent.runs, tt.wantRuns)
|
||||
}
|
||||
if len(ledger.finished) != tt.wantFinished {
|
||||
t.Errorf("claims finished = %d, want %d", len(ledger.finished), tt.wantFinished)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimGateProgressKeepsTheFailure(t *testing.T) {
|
||||
in := "go: downloading github.com/a/b v1.0.0\ngo: downloading github.com/c/d v2.0.0\nprotoc-gen-go: executable file not found in $PATH\nFAIL\n"
|
||||
want := "protoc-gen-go: executable file not found in $PATH\nFAIL\n"
|
||||
if got := trimGateProgress(in); got != want {
|
||||
t.Fatalf("trimGateProgress = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
@@ -61,6 +61,17 @@ func baselineEnvironmentError(argv []string, out, cause string) error {
|
||||
strings.Join(argv, " "), cause, clipTail(out, baselineParkOutputBytes))
|
||||
}
|
||||
|
||||
// baselineSharedError is the park reason for a task that did not repair the
|
||||
// baseline because another task owns that repair.
|
||||
func baselineSharedError(argv []string, out string, claim BaselineClaim) error {
|
||||
verdict := fmt.Sprintf("task %d repaired it on its own branch; retry once that change is merged", claim.Task)
|
||||
if claim.Failure != "" {
|
||||
verdict = fmt.Sprintf("task %d could not repair it: %s", claim.Task, claim.Failure)
|
||||
}
|
||||
return fmt.Errorf("baseline red -- %s fails at this revision and %s\n\ngate output:\n%s",
|
||||
strings.Join(argv, " "), verdict, clipTail(out, baselineParkOutputBytes))
|
||||
}
|
||||
|
||||
// baselineFixFailedError is the park reason for every failure after the
|
||||
// baseline repair agent was dispatched. A failure that names only the agent --
|
||||
// a provider 402, say -- and drops the gate output leaves the operator with no
|
||||
|
||||
@@ -37,10 +37,11 @@ func stageBaselineGateRun(ctx context.Context, tc *TaskContext) error {
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
|
||||
cmd.Dir = tc.Dir
|
||||
out, err := cmd.CombinedOutput()
|
||||
raw, err := cmd.CombinedOutput()
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
out := []byte(trimGateProgress(string(raw)))
|
||||
// An environment failure cannot be repaired in the worktree, so park.
|
||||
if cause := gateEnvironmentCause(string(out), err); cause != "" {
|
||||
return baselineEnvironmentError(argv, string(out), cause)
|
||||
@@ -48,7 +49,7 @@ func stageBaselineGateRun(ctx context.Context, tc *TaskContext) error {
|
||||
tc.Log.WarnContext(ctx, "baseline red -- auto-fixing pre-existing gate failure",
|
||||
"cmd", strings.Join(argv, " "),
|
||||
"err", clipTail(string(out), baselineWarnLogBytes))
|
||||
if fixErr := runBaselineFix(ctx, tc, argv, out); fixErr != nil {
|
||||
if fixErr := repairBaselineOnce(ctx, tc, argv, out); fixErr != nil {
|
||||
return fixErr
|
||||
}
|
||||
}
|
||||
|
||||
@@ -243,7 +243,8 @@ func newGateRepositoryStage(settings yaml.Node) (Stage, error) {
|
||||
func runGateCommand(ctx context.Context, dir string, command agentrun.Command) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, command.Argv[0], command.Argv[1:]...)
|
||||
cmd.Dir = dir
|
||||
out, err := cmd.CombinedOutput()
|
||||
raw, err := cmd.CombinedOutput()
|
||||
out := []byte(trimGateProgress(string(raw)))
|
||||
name := strings.Join(command.Argv, " ")
|
||||
switch {
|
||||
case command.ExpectFailure && err == nil:
|
||||
|
||||
@@ -94,6 +94,9 @@ type TaskContext struct {
|
||||
// to record the step as skipped. It is per-attempt scratch state, reset by
|
||||
// wrapStep before every step.
|
||||
skippedWhen string
|
||||
// baselinePoll is how often a task waits on another's repair; zero uses
|
||||
// the default.
|
||||
baselinePoll time.Duration
|
||||
|
||||
Task *Task
|
||||
Repo config.Repo
|
||||
@@ -102,6 +105,9 @@ type TaskContext struct {
|
||||
Store Store
|
||||
Trees Trees
|
||||
Agent agentrun.Runner
|
||||
// Baseline makes one task repair a red baseline per repository revision.
|
||||
// Nil leaves every task to repair its own.
|
||||
Baseline BaselineLedger
|
||||
// Deliveries records the run's outward effects instead of performing them.
|
||||
// Nil performs them; a test run's RecordDeliveries sets it.
|
||||
Deliveries Deliveries
|
||||
|
||||
@@ -148,6 +148,7 @@ func (t *Transport) EventPublisher() agentexec.EventPublisher { return t.conn }
|
||||
// RemoteTrees is the proxied half of workflow.Trees.
|
||||
type RemoteTrees interface {
|
||||
Push(ctx context.Context) error
|
||||
workflow.BaselineLedger
|
||||
}
|
||||
|
||||
// Forger constructs the identity-scoped forge RPC client.
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package worktreerpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/natsrpc"
|
||||
)
|
||||
|
||||
const (
|
||||
SubjectBaseline = "archie.worktree.baseline"
|
||||
|
||||
// A repair that never reports back (its container died) stops blocking
|
||||
// other tasks after claimTTL; a finished verdict is reused for verdictTTL,
|
||||
// so a transient failure is retried later.
|
||||
claimTTL = time.Hour
|
||||
verdictTTL = 15 * time.Minute
|
||||
)
|
||||
|
||||
type BaselineRequest struct {
|
||||
Grant string `json:"grant"`
|
||||
Key string `json:"key"`
|
||||
Finish bool `json:"finish,omitempty"`
|
||||
Failure string `json:"failure,omitempty"`
|
||||
}
|
||||
|
||||
type BaselineResponse struct {
|
||||
natsrpc.Envelope
|
||||
Lead bool `json:"lead,omitempty"`
|
||||
Task int64 `json:"task,omitempty"`
|
||||
Done bool `json:"done,omitempty"`
|
||||
Failure string `json:"failure,omitempty"`
|
||||
}
|
||||
|
||||
type baselineEntry struct {
|
||||
task int64
|
||||
done bool
|
||||
failure string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// BaselineClaims records which task repairs each red baseline. It is memory
|
||||
// only: after a daemon restart the next task simply claims again.
|
||||
type BaselineClaims struct {
|
||||
mu sync.Mutex
|
||||
entries map[string]baselineEntry
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewBaselineClaims() *BaselineClaims {
|
||||
return &BaselineClaims{entries: map[string]baselineEntry{}, now: time.Now}
|
||||
}
|
||||
|
||||
// Claim makes taskID the repair's owner when nobody holds a live claim for
|
||||
// key; a retry of the owning task takes it back.
|
||||
func (b *BaselineClaims) Claim(taskID int64, key string) workflow.BaselineClaim {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
now := b.now()
|
||||
e, ok := b.entries[key]
|
||||
ttl := claimTTL
|
||||
if e.done {
|
||||
ttl = verdictTTL
|
||||
}
|
||||
if !ok || e.task == taskID || now.Sub(e.at) > ttl {
|
||||
b.entries[key] = baselineEntry{task: taskID, at: now}
|
||||
return workflow.BaselineClaim{Lead: true}
|
||||
}
|
||||
return workflow.BaselineClaim{Task: e.task, Done: e.done, Failure: e.failure}
|
||||
}
|
||||
|
||||
// Finish records the owner's verdict; failure is empty when it repaired the
|
||||
// baseline.
|
||||
func (b *BaselineClaims) Finish(taskID int64, key, failure string) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if e, ok := b.entries[key]; ok && e.task == taskID {
|
||||
b.entries[key] = baselineEntry{task: taskID, done: true, failure: failure, at: b.now()}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handleBaseline(parent context.Context, identity string, msg *nats.Msg) {
|
||||
var req BaselineRequest
|
||||
if err := json.Unmarshal(msg.Data, &req); err != nil {
|
||||
s.respondBaseline(msg, BaselineResponse{Envelope: natsrpc.NewEnvelope(err)})
|
||||
return
|
||||
}
|
||||
ctx, cancel := s.handlerContext(parent)
|
||||
defer cancel()
|
||||
task, err := publishable(ctx, s.Runs, req.Grant, identity)
|
||||
if err != nil || s.Baselines == nil {
|
||||
if err == nil {
|
||||
err = errors.New("baseline claims unavailable")
|
||||
}
|
||||
s.respondBaseline(msg, BaselineResponse{Envelope: natsrpc.NewEnvelope(err)})
|
||||
return
|
||||
}
|
||||
key := task.Owner + "/" + task.Repo + " " + req.Key
|
||||
if req.Finish {
|
||||
s.Baselines.Finish(task.ID, key, req.Failure)
|
||||
s.respondBaseline(msg, BaselineResponse{})
|
||||
return
|
||||
}
|
||||
claim := s.Baselines.Claim(task.ID, key)
|
||||
s.respondBaseline(msg, BaselineResponse{Lead: claim.Lead, Task: claim.Task, Done: claim.Done, Failure: claim.Failure})
|
||||
}
|
||||
|
||||
// ClaimBaselineFix asks the daemon whether this task repairs the red baseline
|
||||
// named by key.
|
||||
func (c *Client) ClaimBaselineFix(ctx context.Context, key string) (workflow.BaselineClaim, error) {
|
||||
resp, err := c.baseline(ctx, BaselineRequest{Grant: c.Grant, Key: key})
|
||||
if err != nil {
|
||||
return workflow.BaselineClaim{}, err
|
||||
}
|
||||
return workflow.BaselineClaim{Lead: resp.Lead, Task: resp.Task, Done: resp.Done, Failure: resp.Failure}, nil
|
||||
}
|
||||
|
||||
// FinishBaselineFix reports the repair's outcome; failure is empty on success.
|
||||
func (c *Client) FinishBaselineFix(ctx context.Context, key, failure string) error {
|
||||
_, err := c.baseline(ctx, BaselineRequest{Grant: c.Grant, Key: key, Finish: true, Failure: failure})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Server) respondBaseline(msg *nats.Msg, resp BaselineResponse) {
|
||||
natsrpc.Respond(msg, s.Log, "worktreerpc", resp)
|
||||
}
|
||||
|
||||
func (c *Client) baseline(ctx context.Context, req BaselineRequest) (BaselineResponse, error) {
|
||||
resp, err := natsrpc.Call[BaselineResponse](ctx, c.rpc(), c.subject(SubjectBaseline), req)
|
||||
if err != nil {
|
||||
return BaselineResponse{}, err
|
||||
}
|
||||
return resp, resp.Err()
|
||||
}
|
||||
@@ -77,6 +77,8 @@ type Server struct {
|
||||
Resolve func(context.Context, *workflow.Task) (*worktree.Manager, error)
|
||||
Runs Runs
|
||||
Log *slog.Logger
|
||||
// Baselines decides which task repairs a red baseline; nil refuses claims.
|
||||
Baselines *BaselineClaims
|
||||
// Timeout bounds a single publication. Zero uses
|
||||
// defaultHandlerTimeout.
|
||||
Timeout time.Duration
|
||||
@@ -93,18 +95,19 @@ func (s *Server) handlerContext(parent context.Context) (context.Context, contex
|
||||
|
||||
// RegisterFor registers publication under the subject scoped to identity.
|
||||
func (s *Server) RegisterFor(nc *nats.Conn, identity string) (unsubscribe func(), err error) {
|
||||
subject := SubjectFor(identity, SubjectPush)
|
||||
subject, baseline := SubjectFor(identity, SubjectPush), SubjectFor(identity, SubjectBaseline)
|
||||
if identity == "*" {
|
||||
subject = "archie.worktree.*.push"
|
||||
subject, baseline = "archie.worktree.*.push", "archie.worktree.*.baseline"
|
||||
}
|
||||
scope := func(msg *nats.Msg) string {
|
||||
if identity == "*" {
|
||||
return strings.Split(msg.Subject, ".")[2]
|
||||
}
|
||||
return identity
|
||||
}
|
||||
return natsrpc.RegisterAll(nc, []natsrpc.Registration{
|
||||
{Subject: subject, Handler: func(ctx context.Context, msg *nats.Msg) {
|
||||
scoped := identity
|
||||
if identity == "*" {
|
||||
scoped = strings.Split(msg.Subject, ".")[2]
|
||||
}
|
||||
s.handlePush(ctx, scoped, msg)
|
||||
}},
|
||||
{Subject: subject, Handler: func(ctx context.Context, msg *nats.Msg) { s.handlePush(ctx, scope(msg), msg) }},
|
||||
{Subject: baseline, Handler: func(ctx context.Context, msg *nats.Msg) { s.handleBaseline(ctx, scope(msg), msg) }},
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user