TUN-3438: move ingress into own package, read into TunnelConfig

cmd/cloudflared/config/configuration.go

@@ -3,6 +3,7 @@ package config
 import (
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -11,6 +12,7 @@ import (
 	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"
 
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/validation"
 )
 
@@ -193,3 +195,17 @@ func ValidateUrl(c *cli.Context, allowFromArgs bool) (string, error) {
 	validUrl, err := validation.ValidateUrl(url)
 	return validUrl, err
 }
+
+func ReadRules(c *cli.Context) ([]ingress.Rule, error) {
+	configFilePath := c.String("config")
+	if configFilePath == "" {
+		return nil, ErrNoConfigFile
+	}
+	fmt.Printf("Reading from config file %s\n", configFilePath)
+	configBytes, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return nil, err
+	}
+	rules, err := ingress.ParseIngress(configBytes)
+	return rules, err
+}

cmd/cloudflared/tunnel/cmd.go

@@ -26,6 +26,7 @@ import (
 	"github.com/cloudflare/cloudflared/dbconnect"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/origin"
@@ -1248,3 +1249,64 @@ reconnect [delay]
 		}
 	}
 }
+
+func buildValidateCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "validate",
+		Action:      cliutil.ErrorHandler(ValidateCommand),
+		Usage:       "Validate the ingress configuration ",
+		UsageText:   "cloudflared tunnel [--config FILEPATH] ingress validate",
+		Description: "Validates the configuration file, ensuring your ingress rules are OK.",
+	}
+}
+
+func buildRuleCommand() *cli.Command {
+	return &cli.Command{
+		Name:      "rule",
+		Action:    cliutil.ErrorHandler(RuleCommand),
+		Usage:     "Check which ingress rule matches a given request URL",
+		UsageText: "cloudflared tunnel [--config FILEPATH] ingress rule URL",
+		ArgsUsage: "URL",
+		Description: "Check which ingress rule matches a given request URL. " +
+			"Ingress rules match a request's hostname and path. Hostname is " +
+			"optional and is either a full hostname like `www.example.com` or a " +
+			"hostname with a `*` for its subdomains, e.g. `*.example.com`. Path " +
+			"is optional and matches a regular expression, like `/[a-zA-Z0-9_]+.html`",
+	}
+}
+
+// Validates the ingress rules in the cloudflared config file
+func ValidateCommand(c *cli.Context) error {
+	_, err := config.ReadRules(c)
+	if err != nil {
+		return errors.Wrap(err, "Validation failed")
+	}
+	if c.IsSet("url") {
+		return ingress.ErrURLIncompatibleWithIngress
+	}
+	fmt.Println("OK")
+	return nil
+}
+
+// Checks which ingress rule matches the given URL.
+func RuleCommand(c *cli.Context) error {
+	rules, err := config.ReadRules(c)
+	if err != nil {
+		return err
+	}
+	requestArg := c.Args().First()
+	if requestArg == "" {
+		return errors.New("cloudflared tunnel rule expects a single argument, the URL to test")
+	}
+	requestURL, err := url.Parse(requestArg)
+	if err != nil {
+		return fmt.Errorf("%s is not a valid URL", requestArg)
+	}
+	if requestURL.Hostname() == "" && requestURL.Scheme == "" {
+		return fmt.Errorf("%s doesn't have a hostname, consider adding a scheme", requestArg)
+	}
+	if requestURL.Hostname() == "" {
+		return fmt.Errorf("%s doesn't have a hostname", requestArg)
+	}
+	return ingress.RuleCommand(rules, requestURL)
+}

cmd/cloudflared/tunnel/configuration.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/tlsconfig"
@@ -184,12 +185,6 @@ func prepareTunnelConfig(
 
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
 
-	originURL, err := config.ValidateUrl(c, compatibilityMode)
-	if err != nil {
-		logger.Errorf("Error validating origin URL: %s", err)
-		return nil, errors.Wrap(err, "Error validating origin URL")
-	}
-
 	var originCert []byte
 	if !isFreeTunnel {
 		originCert, err = getOriginCert(c, logger)
@@ -224,6 +219,36 @@ func prepareTunnelConfig(
 	}
 	dialContext := dialer.DialContext
 
+	var ingressRules []ingress.Rule
+	if namedTunnel != nil {
+		clientUUID, err := uuid.NewRandom()
+		if err != nil {
+			return nil, errors.Wrap(err, "can't generate clientUUID")
+		}
+		namedTunnel.Client = tunnelpogs.ClientInfo{
+			ClientID: clientUUID[:],
+			Features: []string{origin.FeatureSerializedHeaders},
+			Version:  version,
+			Arch:     fmt.Sprintf("%s_%s", buildInfo.GoOS, buildInfo.GoArch),
+		}
+		ingressRules, err = config.ReadRules(c)
+		if err != nil {
+			return nil, err
+		}
+		if c.IsSet("url") {
+			return nil, ingress.ErrURLIncompatibleWithIngress
+		}
+	}
+
+	var originURL string
+	if len(ingressRules) == 0 {
+		originURL, err = config.ValidateUrl(c, compatibilityMode)
+		if err != nil {
+			logger.Errorf("Error validating origin URL: %s", err)
+			return nil, errors.Wrap(err, "Error validating origin URL")
+		}
+	}
+
 	if c.IsSet("unix-socket") {
 		unixSocket, err := config.ValidateUnixSocket(c)
 		if err != nil {
@@ -256,20 +281,6 @@ func prepareTunnelConfig(
 		logger.Errorf("unable to create TLS config to connect with edge: %s", err)
 		return nil, errors.Wrap(err, "unable to create TLS config to connect with edge")
 	}
-
-	if namedTunnel != nil {
-		clientUUID, err := uuid.NewRandom()
-		if err != nil {
-			return nil, errors.Wrap(err, "can't generate clientUUID")
-		}
-		namedTunnel.Client = tunnelpogs.ClientInfo{
-			ClientID: clientUUID[:],
-			Features: []string{origin.FeatureSerializedHeaders},
-			Version:  version,
-			Arch:     fmt.Sprintf("%s_%s", buildInfo.GoOS, buildInfo.GoArch),
-		}
-	}
-
 	return &origin.TunnelConfig{
 		BuildInfo:          buildInfo,
 		ClientID:           clientID,
@@ -301,6 +312,7 @@ func prepareTunnelConfig(
 		TlsConfig:          toEdgeTLSConfig,
 		NamedTunnel:        namedTunnel,
 		ReplaceExisting:    c.Bool("force"),
+		IngressRules:       ingressRules,
 		// turn off use of reconnect token and auth refresh when using named tunnels
 		UseReconnectToken: compatibilityMode && c.Bool("use-reconnect-token"),
 	}, nil

cmd/cloudflared/tunnel/ingress.go

@@ -1,228 +0,0 @@
-package tunnel
-
-import (
-	"fmt"
-	"io/ioutil"
-	"net/url"
-	"regexp"
-	"strings"
-
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-
-	"github.com/pkg/errors"
-	"github.com/urfave/cli/v2"
-	"gopkg.in/yaml.v2"
-)
-
-var (
-	errNoIngressRules      = errors.New("No ingress rules were specified in the config file")
-	errLastRuleNotCatchAll = errors.New("The last ingress rule must match all hostnames (i.e. it must be missing, or must be \"*\")")
-	errBadWildcard         = errors.New("Hostname patterns can have at most one wildcard character (\"*\") and it can only be used for subdomains, e.g. \"*.example.com\"")
-	errNoIngressRulesMatch = errors.New("The URL didn't match any ingress rules")
-)
-
-// Each rule route traffic from a hostname/path on the public
-// internet to the service running on the given URL.
-type rule struct {
-	// Requests for this hostname will be proxied to this rule's service.
-	Hostname string
-
-	// Path is an optional regex that can specify path-driven ingress rules.
-	Path *regexp.Regexp
-
-	// A (probably local) address. Requests for a hostname which matches this
-	// rule's hostname pattern will be proxied to the service running on this
-	// address.
-	Service *url.URL
-}
-
-func (r rule) String() string {
-	var out strings.Builder
-	if r.Hostname != "" {
-		out.WriteString("\thostname: ")
-		out.WriteString(r.Hostname)
-		out.WriteRune('\n')
-	}
-	if r.Path != nil {
-		out.WriteString("\tpath: ")
-		out.WriteString(r.Path.String())
-		out.WriteRune('\n')
-	}
-	out.WriteString("\tservice: ")
-	out.WriteString(r.Service.String())
-	return out.String()
-}
-
-func (r rule) matches(requestURL *url.URL) bool {
-	hostMatch := r.Hostname == "" || r.Hostname == "*" || matchHost(r.Hostname, requestURL.Hostname())
-	pathMatch := r.Path == nil || r.Path.MatchString(requestURL.Path)
-	return hostMatch && pathMatch
-}
-
-func matchHost(ruleHost, reqHost string) bool {
-	if ruleHost == reqHost {
-		return true
-	}
-
-	// Validate hostnames that use wildcards at the start
-	if strings.HasPrefix(ruleHost, "*.") {
-		toMatch := strings.TrimPrefix(ruleHost, "*.")
-		return strings.HasSuffix(reqHost, toMatch)
-	}
-	return false
-}
-
-type unvalidatedRule struct {
-	Hostname string
-	Path     string
-	Service  string
-}
-
-type ingress struct {
-	Ingress []unvalidatedRule
-}
-
-func (ing ingress) validate() ([]rule, error) {
-	rules := make([]rule, len(ing.Ingress))
-	for i, r := range ing.Ingress {
-		service, err := url.Parse(r.Service)
-		if err != nil {
-			return nil, err
-		}
-		if service.Scheme == "" || service.Hostname() == "" {
-			return nil, fmt.Errorf("The service %s must have a scheme and a hostname", r.Service)
-		}
-
-		// Ensure that there are no wildcards anywhere except the first character
-		// of the hostname.
-		if strings.LastIndex(r.Hostname, "*") > 0 {
-			return nil, errBadWildcard
-		}
-
-		// The last rule should catch all hostnames.
-		isCatchAllRule := (r.Hostname == "" || r.Hostname == "*") && r.Path == ""
-		isLastRule := i == len(ing.Ingress)-1
-		if isLastRule && !isCatchAllRule {
-			return nil, errLastRuleNotCatchAll
-		}
-		// ONLY the last rule should catch all hostnames.
-		if !isLastRule && isCatchAllRule {
-			return nil, errRuleShouldNotBeCatchAll{i: i, hostname: r.Hostname}
-		}
-
-		var pathRegex *regexp.Regexp
-		if r.Path != "" {
-			pathRegex, err = regexp.Compile(r.Path)
-			if err != nil {
-				return nil, errors.Wrapf(err, "Rule #%d has an invalid regex", i+1)
-			}
-		}
-
-		rules[i] = rule{
-			Hostname: r.Hostname,
-			Service:  service,
-			Path:     pathRegex,
-		}
-	}
-	return rules, nil
-}
-
-type errRuleShouldNotBeCatchAll struct {
-	i        int
-	hostname string
-}
-
-func (e errRuleShouldNotBeCatchAll) Error() string {
-	return fmt.Sprintf("Rule #%d is matching the hostname '%s', but "+
-		"this will match every hostname, meaning the rules which follow it "+
-		"will never be triggered.", e.i+1, e.hostname)
-}
-
-func parseIngress(rawYAML []byte) ([]rule, error) {
-	var ing ingress
-	if err := yaml.Unmarshal(rawYAML, &ing); err != nil {
-		return nil, err
-	}
-	if len(ing.Ingress) == 0 {
-		return nil, errNoIngressRules
-	}
-	return ing.validate()
-}
-
-func ingressContext(c *cli.Context) ([]rule, error) {
-	configFilePath := c.String("config")
-	if configFilePath == "" {
-		return nil, config.ErrNoConfigFile
-	}
-	fmt.Printf("Reading from config file %s\n", configFilePath)
-	configBytes, err := ioutil.ReadFile(configFilePath)
-	if err != nil {
-		return nil, err
-	}
-	rules, err := parseIngress(configBytes)
-	return rules, err
-}
-
-// Validates the ingress rules in the cloudflared config file
-func validateCommand(c *cli.Context) error {
-	_, err := ingressContext(c)
-	if err != nil {
-		fmt.Println(err.Error())
-		return errors.New("Validation failed")
-	}
-	fmt.Println("OK")
-	return nil
-}
-
-func buildValidateCommand() *cli.Command {
-	return &cli.Command{
-		Name:        "validate",
-		Action:      cliutil.ErrorHandler(validateCommand),
-		Usage:       "Validate the ingress configuration ",
-		UsageText:   "cloudflared tunnel [--config FILEPATH] ingress validate",
-		Description: "Validates the configuration file, ensuring your ingress rules are OK.",
-	}
-}
-
-// Checks which ingress rule matches the given URL.
-func ruleCommand(c *cli.Context) error {
-	rules, err := ingressContext(c)
-	if err != nil {
-		return err
-	}
-	requestArg := c.Args().First()
-	if requestArg == "" {
-		return errors.New("cloudflared tunnel rule expects a single argument, the URL to test")
-	}
-	requestURL, err := url.Parse(requestArg)
-	if err != nil {
-		return fmt.Errorf("%s is not a valid URL", requestArg)
-	}
-	if requestURL.Hostname() == "" {
-		return fmt.Errorf("%s is malformed and doesn't have a hostname", requestArg)
-	}
-	for i, r := range rules {
-		if r.matches(requestURL) {
-			fmt.Printf("Matched rule #%d\n", i+1)
-			fmt.Println(r.String())
-			return nil
-		}
-	}
-	return errNoIngressRulesMatch
-}
-
-func buildRuleCommand() *cli.Command {
-	return &cli.Command{
-		Name:      "rule",
-		Action:    cliutil.ErrorHandler(ruleCommand),
-		Usage:     "Check which ingress rule matches a given request URL",
-		UsageText: "cloudflared [--config CONFIGFILE] tunnel ingress rule URL",
-		ArgsUsage: "URL",
-		Description: "Check which ingress rule matches a given request URL. " +
-			"Ingress rules match a request's hostname and path. Hostname is " +
-			"optional and is either a full hostname like `www.example.com` or a " +
-			"hostname with a `*` for its subdomains, e.g. `*.example.com`. Path " +
-			"is optional and matches a regular expression, like `/[a-zA-Z0-9_]+.html`",
-	}
-}

cmd/cloudflared/tunnel/ingress_test.go

@@ -1,271 +0,0 @@
-package tunnel
-
-import (
-	"net/url"
-	"reflect"
-	"regexp"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func Test_parseIngress(t *testing.T) {
-	localhost8000, err := url.Parse("https://localhost:8000")
-	require.NoError(t, err)
-	localhost8001, err := url.Parse("https://localhost:8001")
-	require.NoError(t, err)
-	type args struct {
-		rawYAML string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    []rule
-		wantErr bool
-	}{
-		{
-			name:    "Empty file",
-			args:    args{rawYAML: ""},
-			wantErr: true,
-		},
-		{
-			name: "Multiple rules",
-			args: args{rawYAML: `
-ingress:
-  - hostname: tunnel1.example.com
-    service: https://localhost:8000
-  - hostname: "*"
-    service: https://localhost:8001
-`},
-			want: []rule{
-				{
-					Hostname: "tunnel1.example.com",
-					Service:  localhost8000,
-				},
-				{
-					Hostname: "*",
-					Service:  localhost8001,
-				},
-			},
-		},
-		{
-			name: "Extra keys",
-			args: args{rawYAML: `
-ingress:
-  - hostname: "*"
-    service: https://localhost:8000
-extraKey: extraValue
-`},
-			want: []rule{
-				{
-					Hostname: "*",
-					Service:  localhost8000,
-				},
-			},
-		},
-		{
-			name: "Hostname can be omitted",
-			args: args{rawYAML: `
-ingress:
-  - service: https://localhost:8000
-`},
-			want: []rule{
-				{
-					Service: localhost8000,
-				},
-			},
-		},
-		{
-			name: "Invalid service",
-			args: args{rawYAML: `
-ingress:
-  - hostname: "*"
-    service: https://local host:8000
-`},
-			wantErr: true,
-		},
-		{
-			name: "Invalid YAML",
-			args: args{rawYAML: `
-key: "value
-`},
-			wantErr: true,
-		},
-		{
-			name: "Last rule isn't catchall",
-			args: args{rawYAML: `
-ingress:
-  - hostname: example.com
-    service: https://localhost:8000
-`},
-			wantErr: true,
-		},
-		{
-			name: "First rule is catchall",
-			args: args{rawYAML: `
-ingress:
-  - service: https://localhost:8000
-  - hostname: example.com
-    service: https://localhost:8000
-`},
-			wantErr: true,
-		},
-		{
-			name: "Catch-all rule can't have a path",
-			args: args{rawYAML: `
-ingress:
-  - service: https://localhost:8001
-    path: /subpath1/(.*)/subpath2
-`},
-			wantErr: true,
-		},
-		{
-			name: "Invalid regex",
-			args: args{rawYAML: `
-ingress:
-  - hostname: example.com
-    service: https://localhost:8000
-    path: "*/subpath2"
-  - service: https://localhost:8001
-`},
-			wantErr: true,
-		},
-		{
-			name: "Service must have a scheme",
-			args: args{rawYAML: `
-ingress:
-  - service: localhost:8000
-`},
-			wantErr: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := parseIngress([]byte(tt.args.rawYAML))
-			if (err != nil) != tt.wantErr {
-				t.Errorf("parseIngress() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("parseIngress() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func MustParse(t *testing.T, rawURL string) *url.URL {
-	u, err := url.Parse(rawURL)
-	require.NoError(t, err)
-	return u
-}
-
-func Test_rule_matches(t *testing.T) {
-	type fields struct {
-		Hostname string
-		Path     *regexp.Regexp
-		Service  *url.URL
-	}
-	type args struct {
-		requestURL *url.URL
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   bool
-	}{
-		{
-			name: "Just hostname, pass",
-			fields: fields{
-				Hostname: "example.com",
-			},
-			args: args{
-				requestURL: MustParse(t, "https://example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Entire hostname is wildcard, should match everything",
-			fields: fields{
-				Hostname: "*",
-			},
-			args: args{
-				requestURL: MustParse(t, "https://example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Just hostname, fail",
-			fields: fields{
-				Hostname: "example.com",
-			},
-			args: args{
-				requestURL: MustParse(t, "https://foo.bar"),
-			},
-			want: false,
-		},
-		{
-			name: "Just wildcard hostname, pass",
-			fields: fields{
-				Hostname: "*.example.com",
-			},
-			args: args{
-				requestURL: MustParse(t, "https://adam.example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Just wildcard hostname, fail",
-			fields: fields{
-				Hostname: "*.example.com",
-			},
-			args: args{
-				requestURL: MustParse(t, "https://tunnel.com"),
-			},
-			want: false,
-		},
-		{
-			name: "Just wildcard outside of subdomain in hostname, fail",
-			fields: fields{
-				Hostname: "*example.com",
-			},
-			args: args{
-				requestURL: MustParse(t, "https://www.example.com"),
-			},
-			want: false,
-		},
-		{
-			name: "Wildcard over multiple subdomains",
-			fields: fields{
-				Hostname: "*.example.com",
-			},
-			args: args{
-				requestURL: MustParse(t, "https://adam.chalmers.example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Hostname and path",
-			fields: fields{
-				Hostname: "*.example.com",
-				Path:     regexp.MustCompile("/static/.*\\.html"),
-			},
-			args: args{
-				requestURL: MustParse(t, "https://www.example.com/static/index.html"),
-			},
-			want: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := rule{
-				Hostname: tt.fields.Hostname,
-				Path:     tt.fields.Path,
-				Service:  tt.fields.Service,
-			}
-			if got := r.matches(tt.args.requestURL); got != tt.want {
-				t.Errorf("rule.matches() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}