Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .agents/skills/verify-lane-schedule/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ With every fact TRUE in the code, setting each fact FALSE in the model alone fai

## Coverage record

Sync commit: `71947bcc360301c223b108b1f6f17332199cdb38` (the last commit at which someone confirmed that the model describes this code; only `check.sh --record` moves it).
Sync commit: `9b4b51995bf3437235e64287ae3293a85c435191` (the last commit at which someone confirmed that the model describes this code; only `check.sh --record` moves it).

Every declaration in these files is covered:

Expand Down
14 changes: 3 additions & 11 deletions .agents/skills/verify-lane-schedule/model/fingerprints.txt
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,7 @@ internal/api/handlers_sync.go:const_deprecationHeader 188405d30be79bf4a07cacfd25
internal/api/handlers_sync.go:const_pullUpstreamAcceptedMsg 76a52910111a2580c149b7ea535374dd1196bde9753bf900ed6667fc5763bdb4
internal/api/handlers_sync.go:const_pushUpstreamAcceptedMsg a2ae388107c13efba0294df359ec78e2d5b92bbb2774f068e5507891b5fea698
internal/api/handlers_sync.go:const_upstreamAppAccessMsg e754e2f035f781ed53cc2ee4cf96a4ee8988a87fa93954aca9989c2040b75599
internal/api/handlers_sync.go:curlIPv4Name 3a642528f1cbc1e513c3d9280e277ce371ba7208b1f880b54b9b851fb92ae32b
internal/api/handlers_sync.go:curlIPv4Part 0d60932194160d2f0f58fc5f0ae6295b84337e94d3a2552565f4ad398cc7cbb4
internal/api/handlers_sync.go:deprecated 3227984a4c52242a8804346ff26305a40483c81f04ead54d0ab913daa01daa24
internal/api/handlers_sync.go:dialableAuthority 272fc206a7652abebab23cc8b90a106a7a2f4571277f9c52ae0ef0b63f3397bc
internal/api/handlers_sync.go:dialableHostname 4aae5d4917b5a62b0b9483a694a95d585a56913a1756a60835291e61d5eb5880
internal/api/handlers_sync.go:import_"bytes" 151fc787b0679fc8767e367d1971b9f907660a3d72d8fa3888877ef9f8ebd9c3
internal/api/handlers_sync.go:import_"cmp" 6ee637994d3b529ead32b904453999f4347fa28f1bba3d141f490cbf02e1f7cc
internal/api/handlers_sync.go:import_"context" 35d620b75b6b66f8c228f26b970a741d77137e4e691aa896119dd9d000eaadd2
Expand All @@ -85,9 +81,7 @@ internal/api/handlers_sync.go:import_"github.com/coder/openstorage/internal/meta
internal/api/handlers_sync.go:import_"github.com/coder/openstorage/internal/sync" 3b3f0175cf3dee88544af1c7af3be7e737e79ae40f672de5cc2c6cc7e6d1f40e
internal/api/handlers_sync.go:import_"io" 93c8d110e63e2ae040bf1dfd2ae6e7c4b4ff1bdc806be48a9ae34456827e0486
internal/api/handlers_sync.go:import_"net/http" 480be5af96d08206ab18c2509cb88ca7831e506d1256768f81e9dd07ebf030a8
internal/api/handlers_sync.go:import_"net/netip" ee93e0a6774d156ffb7dee8e4b29220dbe7dc488e036f596c8ae70f318ffc70e
internal/api/handlers_sync.go:import_"net/url" dbfdd047eb1b2a38b79ee147694d9d1308190f3c019908fa1ebf5e3fd13e2343
internal/api/handlers_sync.go:import_"strconv" 4259f4eb8a51f255fe15fe2b5dea61adc7a54b5df7345920fed692a971b4af46
internal/api/handlers_sync.go:import_"strings" 347f34c3a37316b52e87798b22bb4c8e6173f20f936b730bf65dc5ce6c1fb407
internal/api/handlers_sync.go:import_"time" 9dae0539db6e1e0ec43a0c7ef2211f001e3ddc574b4fc19244eab79b05e23f5a
internal/api/handlers_sync.go:isStrictPullBody ec4bff99487fd47a4c11c3cc2714cee70b94035771aa148211f4abbfd7e798f5
Expand All @@ -96,7 +90,6 @@ internal/api/handlers_sync.go:parseUpstreamConfig dd27982847b44628b2a14a4364e4a2
internal/api/handlers_sync.go:policyDeniesPullUpstream 098960c7bef537c91ca905ce894432882eef7ef465a3ddb0b0f9e7224be746e7
internal/api/handlers_sync.go:refPolicyOpDenies 14c74420941dbb32b29b6bccd73d4c8029dc794fba6a8df30beaf91104b7c0f1
internal/api/handlers_sync.go:registerSyncRoutes 30a124203d6812c4b8fa939a4866caae6a74ac2b0b71e36a7de2fed3e0c25134
internal/api/handlers_sync.go:resolverName 3e71c7fdad4e07921f939372c51f47fe4662e0140f0f73ee453bb27eee32f5b3
internal/api/handlers_sync.go:sentHost 06d61ad20c0eda8ee6d85423eba7a886af4d12740a62517242e024cb01e1381d
internal/api/handlers_sync.go:syncServer.answerTakenSyncName 22cf55fac3487ceec250be953518f3cd9b54265fa9061a6f94c4dd2d8567b0e5
internal/api/handlers_sync.go:syncServer.completeOwnSyncCreate a301e0bad45ed03f793aa89eca3036f4e652051a33060812075c97ccf1076b16
Expand All @@ -118,11 +111,10 @@ internal/api/handlers_sync.go:type_usyncBaseRepo fa63c2d8bb62db2a56acff9a8de55e5
internal/api/handlers_sync.go:type_usyncBaseRepoAuth cbce39e3f3df0443cfcb499a9e439e361335ee86e6e97c205c21b18b48141cd6
internal/api/handlers_sync.go:type_usyncPullUpstreamRequest b98132ea1ae7690f3f00aece7dd8fb2a00f69d0e9ea5f1f95c198c983cc18b0b
internal/api/handlers_sync.go:type_usyncPushUpstreamRequest 3ad57ba2d21ad394741cb6b3ced05a273c72b38ca84d7a73b34d79b4510f4284
internal/api/handlers_sync.go:upstreamConfig.validate 4c75ed7ea1b26bcef15daa0358e3d79ee704eb27b8d1fbdb9861d7a0d4f2f509
internal/api/handlers_sync.go:upstreamConfig.validate 92dd3c7bf8e33cf7987a72b13d5c3b2b28745032f8a945ebee8ccfa7597819af
internal/api/handlers_sync.go:upstreamHostname f65b41a625e2606e71c567cbc6d99d991e59b6ef4da9e976294b9ff3895aac34
internal/api/handlers_sync.go:upstreamURLProblem e66d2c29ef48b6891a62cca5ace9afbd52d579f5f4ae11aeba3cc61b3f3c488d
internal/api/handlers_sync.go:validUpstreamHost 2aa28ab36ae31c94b9311a490f3bfb3707653ce95f7995727c31a37f58cf3c30
internal/api/handlers_sync.go:validateSyncBaseRepo df7737d1b71ebccd3220b074e9002ee36edc83b549338e321394c012a9a6a807
internal/api/handlers_sync.go:validateSyncBaseRepo 53ae861e7da72aa70b924740aaf81d91c6be1c0644b83a47c1dc3106960628dd
internal/api/handlers_sync.go:withStrictPullBody 449923a2f639f42183416d391cac81b304bcf8b6cda9d207474e66e16178766f
internal/api/handlers_sync.go:withUpstreamDetach b16659bb293b5f9eb05b5ade652dfe4fb224e8a7b6c475829007ee1933a164ec
internal/events/refstore.go:BaseRepoID d45f84f8f7f8b9e27d21c7b17553527e76db8c3a5fa34718d2768840b351c8ae
Expand Down Expand Up @@ -834,7 +826,7 @@ internal/sync/worker.go:type_RepoInfo e8a02330a52c30199fa31037b4286d8d23ab173f23
internal/sync/worker.go:type_RepoLookup 819f1bc4b699b113c5c1400e9137b05dd20a8fa12cd1d0d621cca9ac686e29df
internal/sync/worker.go:type_Worker 02235720d66907846dcf03ab0427a9eec32fb417af3c0462bbf5a381c76dfdf1
internal/sync/worker.go:type_syncFailure 6f04984a8621d6e7bf3665dd7af1eb8f7634d923899e5e61b18c60e246c3c59c
internal/sync/worker.go:upstreamURLFailure a4f2c466197974a038dfc2edd51f12da58db9bb332df8709573a6fbd3101cd20
internal/sync/worker.go:upstreamURLFailure f3cab3222d358d74d09d01aa01bf85a19919c4493a8d010c79af0eee15a8c9f3
internal/sync/worker.go:var_authFailureMarkers 28df1810afcf9a0f750de4a7b73b0a925f66ba08df738a9c43a062abc68c8d40
internal/sync/worker.go:var_errClaimStolen 5869b81a47a71928975f5577f5b485bef2e6d0d98f3fd0f31a81b191a67f74a6
internal/sync/worker.go:var_errDetached 09503fc22b7999840131138daf5a27e594cb8b6773f3be5b2001b2bbb0d356e2
Expand Down
12 changes: 12 additions & 0 deletions docs/reference/http-api.md
Original file line number Diff line number Diff line change
Expand Up @@ -953,6 +953,18 @@ pass the total limit together: one git start can carry the URL twice and both cr
entries, about 512 KiB at the limits, so a server with a stack limit of 2 MiB or less
can fail such a run as a failed fetch or push.

A pull or push run whose stored host `POST /repos` refuses (`is not a valid host`,
[Repository lifecycle](#repository-lifecycle-repowrite--orgread)) also fails as
`repository configuration error` before git starts. The server's `WARN` line for the
failed run gives the cause `upstream host is not a valid host` and does not repeat the
host. Only a link stored before `POST /repos` refused its host (by an older release, or
by an older server during an upgrade) has one, such as a name with letters outside ASCII
(`bücher.example`), a number that curl reads as an IPv4 address (`2130706433`), or a
base URL with an upper-case scheme (`HTTPS://`). Such a link fails every run, and no
route changes the host of a stored link: to sync that upstream, create a sync repository
through `POST /repos` with the host in a form it takes, such as `xn--bcher-kva.example`,
the ASCII form of `bücher.example`.

#### Deprecation header

Every response of the four deprecated routes carries `Deprecation: @1791331200`:
Expand Down
2 changes: 1 addition & 1 deletion internal/api/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ The REST API (`/api/v1/...`) and its `/api/repos/{repo_name}` alias layer. `New`
- Merges need git >= 2.38 and refuse unrelated histories unless allowed; a stale target SHA is 409 [handlers_write_merge.go:wmCheckGitVersion] [handlers_write_merge.go:handleMerge]
- Fork and copy check the source token for `git:read` and the pack size against quota before creating any row [handlers_fork.go:handleForkRepo]
- `publishPack` denies over-quota packs before upload and fails closed on a second stale usage floor [publish.go:publishPack]
- `validUpstreamHost` rejects userinfo, query, fragment (even an empty one), a space, a scheme other than lower-case `http`/`https`, a port outside 1-65535, and a hostname that is neither an IP address without a zone nor a name that Go's resolver asks DNS for and that curl does not read as a numerical IPv4 address (one to four decimal, octal or hex numbers within IPv4's range) [handlers_sync.go:validUpstreamHost] [handlers_sync.go:dialableHostname] [handlers_sync.go:curlIPv4Name]
- A sync create refuses a host that `upsync.ValidUpstreamHost` refuses, with the reason `upsync.InvalidUpstreamHostReason` (`is not a valid host`) for `upstream.host` or `base_repo.upstream_host` [handlers_sync.go:upstreamConfig.validate] [handlers_sync.go:validateSyncBaseRepo]
- Text the request asks to store answers 400 on every backend when it holds a NUL byte (a JSON body decodes invalid UTF-8 to U+FFFD, so only a NUL reaches the check): a webhook `secret`, a credential `username` and `password`, the `owner`, `name` and `auth.token` of a sync link, and a `pull-upstream` `ref`. A webhook or credential ID with a NUL byte or invalid UTF-8 answers the 404 of an unknown ID [errors.go:invalidTextFields] [handlers_webhooks.go:handleCreate] [handlers_credentials.go:handleCreateCredential] [handlers_sync.go:validateSyncBaseRepo] [handlers_sync.go:parseUpstreamConfig] [handlers_sync.go:handlePullUpstream]
- A request value that git takes as one argument or one environment entry answers 400 before git starts when it does not fit `gitlimit.MaxArgBytes` with what the server puts around it: a read `path` or grep `paths` entry (`maxGitPathBytes`), a blame `range` and a grep `query.pattern`, and the `name` and `email` of an author, a committer or a note author (`maxGitIdentBytes`). A commit message goes to `commit-tree` on stdin. A list of such values (the grep `paths`, the diff `path` params, the blame `range` params) also answers 400 when its entries do not fit `gitlimit.MaxArgListBytes` together [handlers_read_query.go:listTooLong] [handlers_read_core.go:readPathOK] [handlers_read_query.go:cleanPathParams] [handlers_read_query.go:handleBlame] [handlers_read_query.go:handleGrep] [handlers_read_query.go:grepPathProblems] [handlers_write_commit.go:wcAuthorProblems] [handlers_write_refs.go:wrefHandleNote] [handlers_write_commit.go:wcScratch.writeCommit]
- A sync link or credential that the sync worker could not hand to git answers 400 when it is stored: an upstream URL (`upsync.Upstream.CloneURL`) over `upsync.MaxUpstreamURLBytes`, naming the owner, the name and the host field the body sent, and a `base_repo.auth.token`, credential `username` or `password` over `upsync.MaxCredentialBytes`. The worker checks the same limits again for values stored before [handlers_sync.go:upstreamURLProblem] [handlers_sync.go:validateSyncBaseRepo] [handlers_credentials.go:gcredValidate]
Expand Down
170 changes: 6 additions & 164 deletions internal/api/handlers_sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,7 @@ import (
"fmt"
"io"
"net/http"
"net/netip"
neturl "net/url"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -152,8 +150,8 @@ func validateSyncBaseRepo(spec *usyncBaseRepo, branchErr error, errs *bodyErrors
var hostErr error
host = spec.UpstreamHost
switch {
case host != "" && !validUpstreamHost(host):
hostErr = newFieldError("base_repo.upstream_host", "is not a valid host", "invalid base_repo upstream_host "+errtext.Quote("", spec.UpstreamHost))
case host != "" && !upsync.ValidUpstreamHost(host):
hostErr = newFieldError("base_repo.upstream_host", upsync.InvalidUpstreamHostReason, "invalid base_repo upstream_host "+errtext.Quote("", spec.UpstreamHost))
case host == "" && known:
if host = defaultHost; host == "" {
hostErr = newFieldError("base_repo.upstream_host", "is required for this provider",
Expand Down Expand Up @@ -322,8 +320,8 @@ func (cfg upstreamConfig) validate(errs *bodyErrors) (host string) {
}
host = cfg.Host
switch {
case host != "" && !validUpstreamHost(host):
hostErr = newFieldError("upstream.host", "is not a valid host", "invalid upstream host "+errtext.Quote("", cfg.Host))
case host != "" && !upsync.ValidUpstreamHost(host):
hostErr = newFieldError("upstream.host", upsync.InvalidUpstreamHostReason, "invalid upstream host "+errtext.Quote("", cfg.Host))
case host == "" && known:
if host = defaultHost; host == "" {
hostErr = newFieldError("upstream.host", "is required for this provider", "upstream host is required for provider "+errtext.Quote("", cfg.Provider))
Expand All @@ -338,164 +336,8 @@ func (cfg upstreamConfig) validate(errs *bodyErrors) (host string) {
return host
}

// validUpstreamHost accepts the two documented upstream_host shapes — a
// bare hostname[:port] or an http(s):// base URL — and nothing else. Both
// need a hostname: upstreamHostname asserts one, so a host such as ":80"
// must be refused here and not reach it.
// upsync.CloneURL uses the value verbatim as the base of a git fetch URL,
// so file://, ssh://, transport helpers (ext::), embedded credentials, and
// query/fragment noise must be rejected here, where the attacker-controlled
// value enters (the worker additionally pins GIT_ALLOW_PROTOCOL).
//
// It also refuses hosts that a sync run cannot use as given, so the create
// answers 400 instead of storing a link that the registration check could
// not vet: a hostname that is neither an IP address nor a name that the
// run's resolver asks DNS for (dialableHostname), a port outside 1-65535,
// and a base URL with a query or a fragment, even an empty one
// (`https://h?`), which would end the path of every fetch URL, with a
// space, which curl refuses, or with a scheme that is not lower case
// (`HTTPS://`), which GIT_ALLOW_PROTOCOL refuses.
func validUpstreamHost(host string) bool {
assertf(host != "", "validUpstreamHost: empty host")
if !strings.Contains(host, "://") {
// Bare host[:port]: must round-trip as exactly the authority
// component (no path, userinfo, or control characters). A zone
// is written %25, which the parse decodes, so it never
// round-trips.
u, err := neturl.Parse("https://" + host)
return err == nil && u.Host == host && u.User == nil &&
u.Path == "" && u.RawQuery == "" && u.Fragment == "" && dialableAuthority(u)
}
if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") {
return false
}
u, err := neturl.Parse(host)
if err != nil {
return false
}
// A path is allowed (sub-path-hosted forges); credentials belong in
// the credential store, never in the URL. url.Parse drops an empty
// query or fragment and keeps a space in a path, so the bytes are
// checked.
return u.User == nil && !strings.ContainsAny(host, "?# ") && dialableAuthority(u)
}

// dialableAuthority reports whether the host and port of u name something
// that a sync run can dial: a dialableHostname and, when u has one, a port
// from 1 to 65535. An empty port (`forge.example:`) is the default one.
func dialableAuthority(u *neturl.URL) bool {
if port := u.Port(); port != "" {
if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 {
return false
}
}
return dialableHostname(u.Hostname())
}

// dialableHostname reports whether name is an IP address without a zone,
// or a name that the resolver of a sync run asks DNS for and that curl
// does not read as an IPv4 address (curlIPv4Name). A zone-scoped address
// (fe80::1%eth0) is refused: the egress guard refuses it under every
// policy, and the egress answer would repeat its zone, which only the
// upstream URL limit bounds.
func dialableHostname(name string) bool {
if ip, err := netip.ParseAddr(name); err == nil {
return ip.Zone() == ""
}
return resolverName(name) && !curlIPv4Name(name)
}

// resolverName reports whether the Go resolver of a sync run asks DNS for
// name. A run dials through the egress proxy (upsync.startEgressProxy),
// whose resolver refuses any other name (net.isDomainName, after RFC 1035
// and RFC 3696): at most 253 bytes, or 254 with a final dot; labels of 1 to
// 63 letters, digits, '_' and '-' that do not start or end with '-'; and
// not only digits and dots, which curl reads as an IPv4 address
// (2130706433 is 127.0.0.1) that the registration check never saw. The
// resolver reads /etc/hosts before it applies the rule, but a name that
// only /etc/hosts holds is no upstream host.
//
// A name with other letters goes in its ASCII form
// (xn--bcher-kva.example): the registration check cannot look bücher.example
// up, and whether git and curl convert it depends on the locale that the
// server's git runs in.
func resolverName(name string) bool {
if name == "" || len(name) > 254 || len(name) == 254 && name[253] != '.' {
return false
}
prev := byte('.')
letters := false // a letter, '_' or '-': all digits would be an address
label := 0
for i := 0; i < len(name); i++ {
c := name[i]
switch {
case 'a' <= c && c <= 'z', 'A' <= c && c <= 'Z', c == '_':
letters = true
label++
case '0' <= c && c <= '9':
label++
case c == '-':
if prev == '.' {
return false
}
letters = true
label++
case c == '.':
if prev == '.' || prev == '-' || label > 63 {
return false
}
label = 0
default:
return false
}
prev = c
}
return prev != '-' && label <= 63 && letters
}

// curlIPv4Name reports whether curl reads name as an IPv4 address, as its
// URL parser does (ipv4_normalize): one to four labels, a final dot aside,
// each a decimal, octal (leading 0) or hexadecimal (0x) number of at most
// 32 bits, all but the last at most 255, and the last within the bits the
// others leave, such as 2130706433, 0x7f000001, 0x7f.1 or 1.2.3.0x4. curl
// dials such a name as the address (127.0.0.1, 1.2.3.4), which the
// registration check, which reads it as a name, never saw. A name that only
// looks numeric, such as 0x100.1 or 0x100000000, stays a name for curl.
func curlIPv4Name(name string) bool {
labels := strings.Split(strings.TrimSuffix(name, "."), ".")
if len(labels) > 4 {
return false
}
last := len(labels) - 1
for i, label := range labels {
v, ok := curlIPv4Part(label)
if !ok || i < last && v > 0xff || i == last && v > 0xffffffff>>(8*last) {
return false
}
}
return true
}

// curlIPv4Part parses one label of a numerical IPv4 address as curl does:
// 0x or 0X and hexadecimal digits, 0 and octal digits, or decimal digits,
// at most 32 bits.
func curlIPv4Part(label string) (uint64, bool) {
base, digits := 10, label
switch {
case len(label) >= 2 && label[0] == '0' && (label[1] == 'x' || label[1] == 'X'):
base, digits = 16, label[2:]
case len(label) >= 1 && label[0] == '0':
base = 8
}
if digits == "" {
return 0, false
}
v, err := strconv.ParseUint(digits, base, 32)
return v, err == nil
}

// upstreamHostname extracts the hostname the sync fetch will dial from a
// validUpstreamHost-validated value (bare host[:port] or http(s):// base
// upstreamHostname extracts the hostname the sync fetch will dial from an
// upsync.ValidUpstreamHost-validated value (bare host[:port] or http(s):// base
// URL) for the registration-time egress courtesy check (issue #139).
func upstreamHostname(host string) string {
assertf(host != "", "upstreamHostname: empty host")
Expand Down
Loading
Loading