diff --git a/antrea.yml b/antrea.yml new file mode 100644 index 00000000000..29c8f9e75a8 --- /dev/null +++ b/antrea.yml @@ -0,0 +1,4962 @@ +--- +# Source: crds/antreaagentinfo.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: antreaagentinfos.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1beta1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + additionalPrinterColumns: + - description: Health status of this Agent + jsonPath: ".agentConditions[?(@.type=='AgentHealthy')].status" + name: Healthy + type: string + - description: Last time the Healthy Condition was updated + jsonPath: ".agentConditions[?(@.type=='AgentHealthy')].lastHeartbeatTime" + name: Last Heartbeat + type: date + - description: Version of this Agent + jsonPath: ".version" + name: Version + type: string + priority: 1 + - description: Node on which this Agent is running + jsonPath: ".nodeRef.name" + name: Node + type: string + priority: 1 + - description: Number of local Pods managed by this Agent + jsonPath: ".localPodNum" + name: Num Pods + type: integer + priority: 2 + - description: Subnets used by this Agent for Pod IPAM + jsonPath: ".nodeSubnets" + name: Subnets + type: string + priority: 2 + scope: Cluster + names: + plural: antreaagentinfos + singular: antreaagentinfo + kind: AntreaAgentInfo + shortNames: + - aai + +--- +# Source: crds/antreacontrollerinfo.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: antreacontrollerinfos.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1beta1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + additionalPrinterColumns: + - description: Health status of the Controller + jsonPath: ".controllerConditions[?(@.type=='ControllerHealthy')].status" + name: Healthy + type: string + - description: Last time the Healthy Condition was updated + jsonPath: ".controllerConditions[?(@.type=='ControllerHealthy')].lastHeartbeatTime" + name: Last Heartbeat + type: date + - description: Version of the Controller + jsonPath: ".version" + name: Version + type: string + priority: 1 + - description: Number of Agents connected to the Controller + jsonPath: ".connectedAgentNum" + name: Connected Agents + type: integer + priority: 1 + - description: Node on which the Controller is running + jsonPath: ".nodeRef.name" + name: Node + type: string + priority: 1 + - description: Number of Network Policies computed by Controller + jsonPath: ".networkPolicyControllerInfo.networkPolicyNum" + name: Num Network Policies + type: integer + priority: 2 + scope: Cluster + names: + plural: antreacontrollerinfos + singular: antreacontrollerinfo + kind: AntreaControllerInfo + shortNames: + - aci + +--- +# Source: crds/clustergroup.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clustergroups.crd.antrea.io + labels: + app: antrea + served-by: antrea-controller +spec: + group: crd.antrea.io + versions: + - name: v1alpha2 + served: false + storage: false + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + childGroups: + type: array + items: + type: string + podSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + namespaceSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + externalEntitySelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + ipBlock: + type: object + properties: + cidr: + type: string + format: cidr + ipBlocks: + type: array + items: + type: object + properties: + cidr: + type: string + format: cidr + serviceReference: + type: object + properties: + name: + type: string + namespace: + type: string + status: + type: object + properties: + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + lastTransitionTime: + type: string + - name: v1alpha3 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + childGroups: + type: array + items: + type: string + podSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + namespaceSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + externalEntitySelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + ipBlocks: + type: array + items: + type: object + properties: + cidr: + type: string + format: cidr + serviceReference: + type: object + properties: + name: + type: string + namespace: + type: string + status: + type: object + properties: + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + lastTransitionTime: + type: string + subresources: + status: {} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: ["v1", "v1beta1"] + clientConfig: + service: + name: "antrea" + namespace: "kube-system" + path: "/convert/clustergroup" + scope: Cluster + names: + plural: clustergroups + singular: clustergroup + kind: ClusterGroup + shortNames: + - cg + +--- +# Source: crds/clusternetworkpolicy.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusternetworkpolicies.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - name: Tier + type: string + description: The Tier to which this ClusterNetworkPolicy belongs to. + jsonPath: .spec.tier + - name: Priority + type: number + format: float + description: The Priority of this ClusterNetworkPolicy relative to other policies. + jsonPath: .spec.priority + - name: Desired Nodes + type: number + format: int32 + description: The total number of Nodes that should realize the NetworkPolicy. + jsonPath: .status.desiredNodesRealized + - name: Current Nodes + type: number + format: int32 + description: The number of Nodes that have realized the NetworkPolicy. + jsonPath: .status.currentNodesRealized + - name: Age + type: date + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + properties: + spec: + # Ensure that Spec.Priority field is set + required: + - priority + type: object + properties: + tier: + type: string + priority: + type: number + format: float + # Ensure that Spec.Priority field is between 1 and 10000 + minimum: 1.0 + maximum: 10000.0 + appliedTo: + type: array + items: + type: object + # Ensure that Spec.AppliedTo does not allow IPBlock field + properties: + podSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + namespaceSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + group: + type: string + serviceAccount: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + ingress: + type: array + items: + type: object + required: + - action + properties: + appliedTo: + type: array + items: + type: object + # Ensure that rule AppliedTo does not allow IPBlock field + properties: + podSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + namespaceSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + group: + type: string + serviceAccount: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + service: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + # Ensure that Action field allows only ALLOW, DROP, REJECT and PASS values + action: + type: string + enum: ['Allow', 'Drop', 'Reject', 'Pass'] + ports: + type: array + items: + type: object + properties: + protocol: + type: string + enum: ['TCP', 'UDP', 'SCTP'] + port: + x-kubernetes-int-or-string: true + endPort: + type: integer + sourcePort: + type: integer + sourceEndPort: + type: integer + protocols: + type: array + items: + type: object + oneOf: + - required: [icmp] + - required: [igmp] + properties: + icmp: + type: object + properties: + icmpType: + type: integer + minimum: 0 + maximum: 255 + icmpCode: + type: integer + minimum: 0 + maximum: 255 + igmp: + type: object + properties: + igmpType: + type: integer + # Only IGMP query (0x11) is valid igmpType in ingress rules. + enum: [ 0x11 ] + groupAddress: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + l7Protocols: + type: array + items: + type: object + oneOf: + - required: [http] + properties: + http: + type: object + properties: + host: + type: string + method: + type: string + enum: ['GET', 'POST', 'PUT', 'HEAD', 'DELETE', 'TRACE', 'OPTIONS', 'CONNECT', 'PATCH'] + path: + type: string + from: + type: array + items: + type: object + properties: + podSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + namespaceSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + namespaces: + type: object + properties: + match: + enum: + - Self + type: string + ipBlock: + type: object + properties: + cidr: + type: string + format: cidr + group: + type: string + serviceAccount: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + nodeSelector: + type: object + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + type: array + type: object + type: array + matchLabels: + x-kubernetes-preserve-unknown-fields: true + scope: + type: string + enum: ['Cluster', 'ClusterSet'] + name: + type: string + enableLogging: + type: boolean + logLabel: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]{0,61})?[A-Za-z0-9])?$" + egress: + type: array + items: + type: object + required: + - action + properties: + appliedTo: + type: array + items: + type: object + # Ensure that rule AppliedTo does not allow IPBlock field + properties: + podSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + namespaceSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + group: + type: string + serviceAccount: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + # Ensure that Action field allows only ALLOW, DROP, REJECT and PASS values + action: + type: string + enum: ['Allow', 'Drop', 'Reject', 'Pass'] + ports: + type: array + items: + type: object + properties: + protocol: + type: string + enum: ['TCP', 'UDP', 'SCTP'] + port: + x-kubernetes-int-or-string: true + endPort: + type: integer + sourcePort: + type: integer + sourceEndPort: + type: integer + protocols: + type: array + items: + type: object + oneOf: + - required: [icmp] + - required: [igmp] + properties: + icmp: + type: object + properties: + icmpType: + type: integer + minimum: 0 + maximum: 255 + icmpCode: + type: integer + minimum: 0 + maximum: 255 + igmp: + type: object + properties: + igmpType: + type: integer + # Only IGMP reports are igmpType in egress rules, + # 0x12 is IGMP report V1, 0x16 is IGMP report v2, 0x22 is IGMP report v3. + # It will match all IGMP report types if igmpType is not set. + enum: [ 0x12, 0x16, 0x22 ] + groupAddress: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + l7Protocols: + type: array + items: + type: object + oneOf: + - required: [http] + properties: + http: + type: object + properties: + host: + type: string + method: + type: string + enum: ['GET', 'POST', 'PUT', 'HEAD', 'DELETE', 'TRACE', 'OPTIONS', 'CONNECT', 'PATCH'] + path: + type: string + to: + type: array + items: + type: object + properties: + podSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + namespaceSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + namespaces: + type: object + properties: + match: + enum: + - Self + type: string + ipBlock: + type: object + properties: + cidr: + type: string + format: cidr + group: + type: string + fqdn: + type: string + serviceAccount: + type: object + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + nodeSelector: + type: object + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + type: array + type: object + type: array + matchLabels: + x-kubernetes-preserve-unknown-fields: true + toServices: + type: array + items: + type: object + required: + - name + - namespace + properties: + name: + type: string + namespace: + type: string + scope: + type: string + enum: [ 'Cluster', 'ClusterSet' ] + name: + type: string + enableLogging: + type: boolean + logLabel: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]{0,61})?[A-Za-z0-9])?$" + status: + type: object + properties: + phase: + type: string + observedGeneration: + type: integer + currentNodesRealized: + type: integer + desiredNodesRealized: + type: integer + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + lastTransitionTime: + type: string + reason: + type: string + message: + type: string + subresources: + status: {} + scope: Cluster + names: + plural: clusternetworkpolicies + singular: clusternetworkpolicy + kind: ClusterNetworkPolicy + shortNames: + - acnp + +--- +# Source: crds/egress.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: egresses.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1alpha2 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + spec: + type: object + required: + - appliedTo + oneOf: + - anyOf: + - required: + - egressIP + - required: + - externalIPPool + - anyOf: + - required: + - egressIPs + - required: + - externalIPPools + properties: + appliedTo: + type: object + properties: + podSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + namespaceSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + egressIP: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + egressIPs: + type: array + items: + type: string + oneOf: + - maxLength: 0 + - format: ipv4 + - format: ipv6 + externalIPPool: + type: string + externalIPPools: + type: array + items: + type: string + status: + type: object + properties: + egressNode: + type: string + egressIP: + type: string + additionalPrinterColumns: + - description: The effective SNAT IP address for the selected workloads. + jsonPath: .status.egressIP + name: EgressIP + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: The Owner Node of egress IP + jsonPath: .status.egressNode + name: Node + type: string + subresources: + status: {} + scope: Cluster + names: + plural: egresses + singular: egress + kind: Egress + shortNames: + - eg + +--- +# Source: crds/externalentity.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: externalentities.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1alpha2 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + endpoints: + type: array + items: + type: object + properties: + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + name: + type: string + ports: + type: array + items: + type: object + properties: + protocol: + type: string + enum: ['TCP', 'UDP', 'SCTP'] + port: + x-kubernetes-int-or-string: true + name: + type: string + externalNode: + type: string + - name: v1alpha1 + served: false + storage: false + schema: + openAPIV3Schema: + type: object + scope: Namespaced + names: + plural: externalentities + singular: externalentity + kind: ExternalEntity + shortNames: + - ee + +--- +# Source: crds/externalippool.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: externalippools.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1alpha2 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + spec: + type: object + required: + - ipRanges + - nodeSelector + properties: + ipRanges: + type: array + items: + type: object + oneOf: + - required: + - cidr + - required: + - start + - end + properties: + cidr: + type: string + format: cidr + start: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + end: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + nodeSelector: + type: object + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + type: array + type: object + type: array + matchLabels: + x-kubernetes-preserve-unknown-fields: true + status: + type: object + properties: + usage: + type: object + properties: + total: + type: integer + used: + type: integer + additionalPrinterColumns: + - description: The number of total IPs + jsonPath: .status.usage.total + name: Total + type: integer + - description: The number of allocated IPs + jsonPath: .status.usage.used + name: Used + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + subresources: + status: {} + scope: Cluster + names: + plural: externalippools + singular: externalippool + kind: ExternalIPPool + shortNames: + - eip + +--- +# Source: crds/externalnode.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + labels: + app: antrea + name: externalnodes.crd.antrea.io +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + spec: + type: object + required: + - interfaces + properties: + interfaces: + type: array + minItems: 1 + maxItems: 1 + required: + - ips + items: + type: object + properties: + ips: + type: array + minItems: 1 + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + name: + type: string + served: true + storage: true + scope: Namespaced + names: + plural: externalnodes + singular: externalnode + kind: ExternalNode + shortNames: + - en + +--- +# Source: crds/ippool.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ippools.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1alpha2 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + spec: + required: + - ipVersion + - ipRanges + type: object + properties: + ipVersion: + type: integer + enum: [ 4, 6 ] + ipRanges: + items: + oneOf: + - required: + - cidr + - gateway + - prefixLength + - required: + - start + - end + - gateway + - prefixLength + properties: + cidr: + format: cidr + type: string + start: + oneOf: + - format: ipv4 + - format: ipv6 + type: string + end: + oneOf: + - format: ipv4 + - format: ipv6 + type: string + gateway: + oneOf: + - format: ipv4 + - format: ipv6 + type: string + prefixLength: + type: integer + minimum: 1 + maximum: 128 + vlan: + type: integer + minimum: 0 + maximum: 4094 + type: object + type: array + status: + properties: + ipAddresses: + items: + properties: + ipAddress: + type: string + owner: + properties: + pod: + properties: + name: + type: string + namespace: + type: string + containerID: + type: string + ifName: + type: string + type: object + statefulSet: + properties: + name: + type: string + namespace: + type: string + index: + type: integer + type: object + type: object + phase: + type: string + type: object + type: array + usage: + properties: + used: + type: integer + total: + type: integer + type: object + type: object + additionalPrinterColumns: + - description: The number of total IPs + jsonPath: .status.usage.total + name: Total + type: integer + - description: The number of allocated IPs + jsonPath: .status.usage.used + name: Used + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + subresources: + status: {} + scope: Cluster + names: + plural: ippools + singular: ippool + kind: IPPool + shortNames: + - ipp + +--- +# Source: crds/networkpolicy.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: networkpolicies.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - name: Tier + type: string + description: The Tier to which this Antrea NetworkPolicy belongs to. + jsonPath: .spec.tier + - name: Priority + type: number + format: float + description: The Priority of this Antrea NetworkPolicy relative to other policies. + jsonPath: .spec.priority + - name: Desired Nodes + type: number + format: int32 + description: The total number of Nodes that should realize the NetworkPolicy. + jsonPath: .status.desiredNodesRealized + - name: Current Nodes + type: number + format: int32 + description: The number of Nodes that have realized the NetworkPolicy. + jsonPath: .status.currentNodesRealized + - name: Age + type: date + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + properties: + spec: + # Ensure that Spec.Priority field is set + required: + - priority + type: object + properties: + tier: + type: string + priority: + type: number + format: float + # Ensure that Spec.Priority field is between 1 and 10000 + minimum: 1.0 + maximum: 10000.0 + appliedTo: + type: array + items: + type: object + # Ensure that Spec.AppliedTo does not allow NamespaceSelector/IPBlock field + properties: + externalEntitySelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + podSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + group: + type: string + ingress: + type: array + items: + type: object + required: + - action + properties: + appliedTo: + type: array + items: + type: object + # Ensure that rule AppliedTo does not allow NamespaceSelector/IPBlock field + properties: + podSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + group: + type: string + # Ensure that Action field allows only ALLOW, DROP, REJECT and PASS values + action: + type: string + enum: ['Allow', 'Drop', 'Reject', 'Pass'] + ports: + type: array + items: + type: object + properties: + protocol: + type: string + enum: ['TCP', 'UDP', 'SCTP'] + port: + x-kubernetes-int-or-string: true + endPort: + type: integer + sourcePort: + type: integer + sourceEndPort: + type: integer + protocols: + type: array + items: + type: object + oneOf: + - required: [icmp] + - required: [igmp] + properties: + icmp: + type: object + properties: + icmpType: + type: integer + minimum: 0 + maximum: 255 + icmpCode: + type: integer + minimum: 0 + maximum: 255 + igmp: + type: object + properties: + igmpType: + type: integer + # Only IGMP query (0x11) is valid igmpType in ingress rules. + enum: [ 0x11 ] + groupAddress: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + l7Protocols: + type: array + items: + type: object + oneOf: + - required: [http] + properties: + http: + type: object + properties: + host: + type: string + method: + type: string + enum: ['GET', 'POST', 'PUT', 'HEAD', 'DELETE', 'TRACE', 'OPTIONS', 'CONNECT', 'PATCH'] + path: + type: string + from: + type: array + items: + type: object + properties: + podSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + namespaceSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + externalEntitySelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + ipBlock: + type: object + properties: + cidr: + type: string + format: cidr + nodeSelector: + type: object + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + type: array + type: object + type: array + matchLabels: + x-kubernetes-preserve-unknown-fields: true + group: + type: string + scope: + type: string + enum: ['Cluster', 'ClusterSet'] + name: + type: string + enableLogging: + type: boolean + logLabel: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]{0,61})?[A-Za-z0-9])?$" + egress: + type: array + items: + type: object + required: + - action + properties: + appliedTo: + type: array + items: + type: object + # Ensure that rule AppliedTo does not allow NamespaceSelector/IPBlock field + properties: + podSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + group: + type: string + # Ensure that Action field allows only ALLOW, DROP, REJECT and PASS values + action: + type: string + enum: ['Allow', 'Drop', 'Reject', 'Pass'] + ports: + type: array + items: + type: object + properties: + protocol: + type: string + enum: ['TCP', 'UDP', 'SCTP'] + port: + x-kubernetes-int-or-string: true + endPort: + type: integer + sourcePort: + type: integer + sourceEndPort: + type: integer + protocols: + type: array + items: + type: object + oneOf: + - required: [icmp] + - required: [igmp] + properties: + icmp: + type: object + properties: + icmpType: + type: integer + minimum: 0 + maximum: 255 + icmpCode: + type: integer + minimum: 0 + maximum: 255 + igmp: + type: object + properties: + igmpType: + type: integer + # Only IGMP reports are igmpType in egress rules, + # 0x12 is IGMP report V1, 0x16 is IGMP report v2, 0x22 is IGMP report v3. + # It will match all IGMP report types if igmpType is not set. + enum: [ 0x12, 0x16, 0x22 ] + groupAddress: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + l7Protocols: + type: array + items: + type: object + oneOf: + - required: [http] + properties: + http: + type: object + properties: + host: + type: string + method: + type: string + enum: ['GET', 'POST', 'PUT', 'HEAD', 'DELETE', 'TRACE', 'OPTIONS', 'CONNECT', 'PATCH'] + path: + type: string + to: + type: array + items: + type: object + properties: + podSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + namespaceSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + externalEntitySelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + ipBlock: + type: object + properties: + cidr: + type: string + format: cidr + fqdn: + type: string + nodeSelector: + type: object + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + type: array + type: object + type: array + matchLabels: + x-kubernetes-preserve-unknown-fields: true + group: + type: string + toServices: + type: array + items: + type: object + required: + - name + properties: + name: + type: string + namespace: + type: string + scope: + type: string + name: + type: string + enableLogging: + type: boolean + logLabel: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]{0,61})?[A-Za-z0-9])?$" + status: + type: object + properties: + phase: + type: string + observedGeneration: + type: integer + currentNodesRealized: + type: integer + desiredNodesRealized: + type: integer + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + lastTransitionTime: + type: string + reason: + type: string + message: + type: string + subresources: + status: {} + scope: Namespaced + names: + plural: networkpolicies + singular: networkpolicy + kind: NetworkPolicy + shortNames: + - anp + +--- +# Source: crds/supportbundlecollection.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: supportbundlecollections.crd.antrea.io +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + required: + - fileServer + - authentication + anyOf: + - required: + - nodes + - required: + - externalNodes + properties: + nodes: + type: object + properties: + nodeNames: + type: array + items: + type: string + nodeSelector: + type: object + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + type: array + type: object + type: array + matchLabels: + x-kubernetes-preserve-unknown-fields: true + externalNodes: + type: object + required: + - namespace + properties: + namespace: + type: string + nodeNames: + type: array + items: + type: string + nodeSelector: + type: object + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + type: array + type: object + type: array + matchLabels: + x-kubernetes-preserve-unknown-fields: true + expirationMinutes: + type: integer + default: 60 + sinceTime: + type: string + format: duration + fileServer: + type: object + properties: + url: + type: string + authentication: + type: object + properties: + authType: + type: string + enum: ["BearerToken", "APIKey", "BasicAuthentication"] + authSecret: + type: object + properties: + name: + type: string + namespace: + type: string + status: + type: object + properties: + collectedNodes: + type: integer + desiredNodes: + type: integer + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + lastTransitionTime: + type: string + reason: + type: string + message: + type: string + subresources: + status: {} + scope: Cluster + names: + plural: supportbundlecollections + singular: supportbundlecollection + kind: SupportBundleCollection + shortNames: + - sbc + +--- +# Source: crds/tier.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: tiers.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - name: Priority + type: integer + description: The Priority of this Tier relative to other Tiers. + jsonPath: .spec.priority + - name: Age + type: date + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + properties: + spec: + required: + - priority + type: object + properties: + priority: + type: integer + minimum: 0 + maximum: 255 + description: + type: string + scope: Cluster + names: + plural: tiers + singular: tier + kind: Tier + shortNames: + - tr + +--- +# Source: crds/traceflow.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: traceflows.crd.antrea.io + labels: + app: antrea +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - jsonPath: .status.phase + description: The phase of the Traceflow. + name: Phase + type: string + - jsonPath: .spec.source.pod + description: The name of the source Pod. + name: Source-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.pod + description: The name of the destination Pod. + name: Destination-Pod + type: string + priority: 10 + - jsonPath: .spec.destination.ip + description: The IP address of the destination. + name: Destination-IP + type: string + priority: 10 + - jsonPath: .spec.liveTraffic + description: Trace live traffic. + name: Live-Traffic + type: boolean + priority: 10 + - jsonPath: .spec.droppedOnly + description: Capture only the dropped packet. + name: Dropped-Only + type: boolean + priority: 10 + - jsonPath: .spec.timeout + description: Timeout in seconds. + name: Timeout + type: integer + priority: 10 + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + spec: + type: object + properties: + source: + type: object + properties: + pod: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + destination: + type: object + properties: + pod: + type: string + service: + type: string + namespace: + type: string + ip: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + packet: + type: object + properties: + ipHeader: + type: object + properties: + srcIP: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + protocol: + type: integer + ttl: + type: integer + flags: + type: integer + ipv6Header: + type: object + properties: + srcIP: + type: string + format: ipv6 + nextHeader: + type: integer + hopLimit: + type: integer + transportHeader: + type: object + properties: + icmp: + type: object + properties: + id: + type: integer + sequence: + type: integer + udp: + type: object + properties: + srcPort: + type: integer + dstPort: + type: integer + tcp: + type: object + properties: + srcPort: + type: integer + dstPort: + type: integer + flags: + type: integer + liveTraffic: + type: boolean + droppedOnly: + type: boolean + timeout: + type: integer + status: + type: object + properties: + reason: + type: string + dataplaneTag: + type: integer + phase: + type: string + startTime: + type: string + results: + type: array + items: + type: object + properties: + node: + type: string + role: + type: string + timestamp: + type: integer + observations: + type: array + items: + type: object + properties: + component: + type: string + componentInfo: + type: string + action: + type: string + pod: + type: string + dstMAC: + type: string + networkPolicy: + type: string + ttl: + type: integer + translatedSrcIP: + type: string + translatedDstIP: + type: string + tunnelDstIP: + type: string + egressIP: + type: string + egress: + type: string + capturedPacket: + properties: + srcIP: + type: string + dstIP: + type: string + length: + type: integer + ipHeader: + properties: + flags: + type: integer + protocol: + type: integer + ttl: + type: integer + type: object + ipv6Header: + properties: + hopLimit: + type: integer + nextHeader: + type: integer + type: object + transportHeader: + properties: + tcp: + properties: + dstPort: + type: integer + srcPort: + type: integer + flags: + type: integer + type: object + udp: + properties: + dstPort: + type: integer + srcPort: + type: integer + type: object + icmp: + properties: + id: + type: integer + sequence: + type: integer + type: object + type: object + type: object + subresources: + status: {} + scope: Cluster + names: + plural: traceflows + singular: traceflow + kind: Traceflow + shortNames: + - tf + +--- +# Source: crds/trafficcontrol.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: trafficcontrols.crd.antrea.io +spec: + group: crd.antrea.io + versions: + - name: v1alpha2 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + required: + - spec + properties: + spec: + type: object + required: + - appliedTo + - direction + - action + - targetPort + properties: + appliedTo: + type: object + properties: + podSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + namespaceSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true + direction: + type: string + enum: + - Ingress + - Egress + - Both + action: + type: string + enum: + - Mirror + - Redirect + targetPort: + type: object + oneOf: + - required: [ovsInternal] + - required: [device] + - required: [geneve] + - required: [vxlan] + - required: [gre] + - required: [erspan] + properties: + ovsInternal: + type: object + required: + - name + properties: + name: + type: string + device: + type: object + required: + - name + properties: + name: + type: string + geneve: + type: object + required: + - remoteIP + properties: + remoteIP: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + vni: + type: integer + minimum: 0 + maximum: 16777215 + destinationPort: + type: integer + minimum: 1 + maximum: 65535 + vxlan: + type: object + required: + - remoteIP + properties: + remoteIP: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + vni: + type: integer + minimum: 0 + maximum: 16777215 + destinationPort: + type: integer + minimum: 1 + maximum: 65535 + gre: + type: object + required: + - remoteIP + properties: + remoteIP: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + key: + type: integer + minimum: 0 + maximum: 4294967295 + erspan: + type: object + required: + - remoteIP + - version + properties: + remoteIP: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + sessionID: + type: integer + minimum: 0 + maximum: 1023 + version: + type: integer + enum: + - 1 + - 2 + index: + type: integer + dir: + type: integer + enum: + - 0 + - 1 + hardwareID: + type: integer + returnPort: + type: object + oneOf: + - required: [ovsInternal] + - required: [device] + - required: [geneve] + - required: [vxlan] + - required: [gre] + properties: + ovsInternal: + type: object + required: + - name + properties: + name: + type: string + device: + type: object + required: + - name + properties: + name: + type: string + geneve: + type: object + required: + - remoteIP + properties: + remoteIP: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + vni: + type: integer + minimum: 0 + maximum: 16777215 + destinationPort: + type: integer + minimum: 1 + maximum: 65535 + vxlan: + type: object + required: + - remoteIP + properties: + remoteIP: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + vni: + type: integer + minimum: 0 + maximum: 16777215 + destinationPort: + type: integer + minimum: 1 + maximum: 65535 + gre: + type: object + required: + - remoteIP + properties: + remoteIP: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + key: + type: integer + minimum: 0 + maximum: 4294967295 + additionalPrinterColumns: + - description: Specifies the direction of traffic that should be matched. + jsonPath: .spec.direction + name: Direction + type: string + - description: Specifies the action that should be taken for the traffic. + jsonPath: .spec.action + name: Action + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + subresources: + status: {} + scope: Cluster + names: + plural: trafficcontrols + singular: trafficcontrol + kind: TrafficControl + shortNames: + - tc + +--- +# Source: antrea/templates/agent/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: antrea-agent + namespace: kube-system + labels: + app: antrea +--- +# Source: antrea/templates/antctl/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: antctl + namespace: kube-system + labels: + app: antrea +--- +# Source: antrea/templates/controller/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: antrea-controller + namespace: kube-system + labels: + app: antrea +--- +# Source: antrea/templates/agent/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: antrea-agent-service-account-token + namespace: kube-system + annotations: + kubernetes.io/service-account.name: antrea-agent +type: kubernetes.io/service-account-token +--- +# Source: antrea/templates/antctl/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: antctl-service-account-token + namespace: kube-system + annotations: + kubernetes.io/service-account.name: antctl +type: kubernetes.io/service-account-token +--- +# Source: antrea/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: antrea-config + namespace: kube-system + labels: + app: antrea +data: + antrea-agent.conf: | + # FeatureGates is a map of feature names to bools that enable or disable experimental features. + featureGates: + # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha. + # AllAlpha: false + + # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta. + # AllBeta: false + + # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent. + # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on + # Service traffic. + # AntreaProxy: true + + # Enable EndpointSlice support in AntreaProxy. Don't enable this feature unless that EndpointSlice + # API version v1 is supported and set as enabled in Kubernetes. If AntreaProxy is not enabled, + # this flag will not take effect. + # EndpointSlice: true + + # Enable TopologyAwareHints in AntreaProxy. This requires AntreaProxy and EndpointSlice to be + # enabled, otherwise this flag will not take effect. + # TopologyAwareHints: true + + # Enable traceflow which provides packet tracing feature to diagnose network issue. + # Traceflow: true + + # Enable NodePortLocal feature to make the Pods reachable externally through NodePort + # NodePortLocal: true + + # Enable Antrea ClusterNetworkPolicy feature to complement K8s NetworkPolicy for cluster admins + # to define security policies which apply to the entire cluster, and Antrea NetworkPolicy + # feature that supports priorities, rule actions and externalEntities in the future. + # AntreaPolicy: true + + # Enable flowexporter which exports polled conntrack connections as IPFIX flow records from each + # agent to a configured collector. + FlowExporter: true + + # Enable collecting and exposing NetworkPolicy statistics. + # NetworkPolicyStats: true + + # Enable controlling SNAT IPs of Pod egress traffic. + # Egress: true + + # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the + # bridging mode and allocates IPs to Pods in bridging mode. It is also required to use Antrea for + # IPAM when configuring secondary network interfaces with Multus. + # AntreaIPAM: false + + # Enable multicast traffic. + # Multicast: true + + # Enable Antrea Multi-cluster features. + # Multicluster: false + + # Enable support for provisioning secondary network interfaces for Pods (using + # Pod annotations). At the moment, Antrea can only create secondary network + # interfaces using SR-IOV VFs on baremetal Nodes. + # SecondaryNetwork: false + + # Enable managing external IPs of Services of LoadBalancer type. + # ServiceExternalIP: false + + # Enable mirroring or redirecting the traffic Pods send or receive. + TrafficControl: true + + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + + # Enable collecting support bundle files with SupportBundleCollection CRD. + # SupportBundleCollection: false + + # Enable users to protect their applications by specifying how they are allowed to communicate with others, taking + # into account application context. + L7NetworkPolicy: true + + # Enable L7Visibility on Pods and Namespace. + L7Visibility: true + + # Name of the OpenVSwitch bridge antrea-agent will create and use. + # Make sure it doesn't conflict with your existing OpenVSwitch bridges. + ovsBridge: "br-int" + + # Datapath type to use for the OpenVSwitch bridge created by Antrea. At the moment, the only + # supported value is 'system', which corresponds to the kernel datapath. + #ovsDatapathType: system + + # Name of the interface antrea-agent will create and use for host <--> pod communication. + # Make sure it doesn't conflict with your existing interfaces. + hostGateway: "antrea-gw0" + + # Determines how traffic is encapsulated. It has the following options: + # encap(default): Inter-node Pod traffic is always encapsulated and Pod to external network + # traffic is SNAT'd. + # noEncap: Inter-node Pod traffic is not encapsulated; Pod to external network traffic is + # SNAT'd if noSNAT is not set to true. Underlying network must be capable of + # supporting Pod traffic across IP subnets. + # hybrid: noEncap if source and destination Nodes are on the same subnet, otherwise encap. + # networkPolicyOnly: Antrea enforces NetworkPolicy only, and utilizes CNI chaining and delegates Pod + # IPAM and connectivity to the primary CNI. + # + trafficEncapMode: "encap" + + # Whether or not to SNAT (using the Node IP) the egress traffic from a Pod to the external network. + # This option is for the noEncap traffic mode only, and the default value is false. In the noEncap + # mode, if the cluster's Pod CIDR is reachable from the external network, then the Pod traffic to + # the external network needs not be SNAT'd. In the networkPolicyOnly mode, antrea-agent never + # performs SNAT and this option will be ignored; for other modes it must be set to false. + noSNAT: false + + # Tunnel protocols used for encapsulating traffic across Nodes. If WireGuard is enabled in trafficEncryptionMode, + # this option will not take effect. Supported values: + # - geneve (default) + # - vxlan + # - gre + # - stt + # Note that "gre" is not supported for IPv6 clusters (IPv6-only or dual-stack clusters). + tunnelType: "geneve" + + # TunnelPort is the destination port for UDP and TCP based tunnel protocols (Geneve, VXLAN, and STT). + # If zero, it will use the assigned IANA port for the protocol, i.e. 6081 for Geneve, 4789 for VXLAN, + # and 7471 for STT. + tunnelPort: 0 + + # TunnelCsum determines whether to compute UDP encapsulation header (Geneve or VXLAN) checksums on outgoing + # packets. For Linux kernel before Mar 2021, UDP checksum must be present to trigger GRO on the receiver for better + # performance of Geneve and VXLAN tunnels. The issue has been fixed by + # https://github.com/torvalds/linux/commit/89e5c58fc1e2857ccdaae506fb8bc5fed57ee063, thus computing UDP checksum is + # no longer necessary. + # It should only be set to true when you are using an unpatched Linux kernel and observing poor transfer performance. + tunnelCsum: false + + # Determines how tunnel traffic is encrypted. Currently encryption only works with encap mode. + # It has the following options: + # - none (default): Inter-node Pod traffic will not be encrypted. + # - ipsec: Enable IPsec (ESP) encryption for Pod traffic across Nodes. Antrea uses + # Preshared Key (PSK) for IKE authentication. When IPsec tunnel is enabled, + # the PSK value must be passed to Antrea Agent through an environment + # variable: ANTREA_IPSEC_PSK. + # - wireGuard: Enable WireGuard for tunnel traffic encryption. + trafficEncryptionMode: "none" + + # Enable bridging mode of Pod network on Nodes, in which the Node's transport interface is connected + # to the OVS bridge, and cross-Node/VLAN traffic of AntreaIPAM Pods (Pods whose IP addresses are + # allocated by AntreaIPAM from IPPools) is sent to the underlay network, and forwarded/routed by the + # underlay network. + # This option requires the `AntreaIPAM` feature gate to be enabled. At this moment, it supports only + # IPv4 and Linux Nodes, and can be enabled only when `ovsDatapathType` is `system`, + # `trafficEncapMode` is `noEncap`, and `noSNAT` is true. + enableBridgingMode: false + + # Disable TX checksum offloading for container network interfaces. It's supposed to be set to true when the + # datapath doesn't support TX checksum offloading, which causes packets to be dropped due to bad checksum. + # It affects Pods running on Linux Nodes only. + disableTXChecksumOffload: true + + # Default MTU to use for the host gateway interface and the network interface of each Pod. + # If omitted, antrea-agent will discover the MTU of the Node's primary interface and + # also adjust MTU to accommodate for tunnel encapsulation overhead (if applicable). + defaultMTU: 0 + + # wireGuard specifies WireGuard related configurations. + wireGuard: + # The port for WireGuard to receive traffic. + port: 51820 + + egress: + # exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNAT'd by Egresses. + exceptCIDRs: + # The maximum number of Egress IPs that can be assigned to a Node. It's useful when the Node network restricts + # the number of secondary IPs a Node can have, e.g. EKS. It must not be greater than 255. + maxEgressIPsPerNode: 255 + + # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be + # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When + # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided. + serviceCIDR: "" + + # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack + # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by + # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. + # No default value for this field. + serviceCIDRv6: "" + + # The port for the antrea-agent APIServer to serve on. + # Note that if it's set to another value, the `containerPort` of the `api` port of the + # `antrea-agent` container must be set to the same value. + apiPort: 10350 + + # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. + enablePrometheusMetrics: true + + + flowExporter: + # Enable FlowExporter, a feature used to export polled conntrack connections as + # IPFIX flow records from each agent to a configured collector. To enable this + # feature, you need to set "enable" to true, and ensure that the FlowExporter + # feature gate is also enabled. + enable: true + # Provide the IPFIX collector address as a string with format :[][:]. + # HOST can either be the DNS name, IP, or Service name of the Flow Collector. If + # using an IP, it can be either IPv4 or IPv6. However, IPv6 address should be + # wrapped with []. When the collector is running in-cluster as a Service, set + # to /. For example, + # "flow-aggregator/flow-aggregator" can be provided to connect to the Antrea + # Flow Aggregator Service. + # If PORT is empty, we default to 4739, the standard IPFIX port. + # If no PROTO is given, we consider "tls" as default. We support "tls", "tcp" and + # "udp" protocols. "tls" is used for securing communication between flow exporter and + # flow aggregator. + flowCollectorAddr: "flow-aggregator/flow-aggregator:4739:tls" + + # Provide flow poll interval as a duration string. This determines how often the + # flow exporter dumps connections from the conntrack module. Flow poll interval + # should be greater than or equal to 1s (one second). + # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + flowPollInterval: "5s" + + # Provide the active flow export timeout, which is the timeout after which a flow + # record is sent to the collector for active flows. Thus, for flows with a continuous + # stream of packets, a flow record will be exported to the collector once the elapsed + # time since the last export event is equal to the value of this timeout. + # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + activeFlowExportTimeout: "5s" + + # Provide the idle flow export timeout, which is the timeout after which a flow + # record is sent to the collector for idle flows. A flow is considered idle if no + # packet matching this flow has been observed since the last export event. + # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + idleFlowExportTimeout: "15s" + + nodePortLocal: + # Enable NodePortLocal, a feature used to make Pods reachable using port forwarding on the host. To + # enable this feature, you need to set "enable" to true, and ensure that the NodePortLocal feature + # gate is also enabled (which is the default). + enable: false + # Provide the port range used by NodePortLocal. When the NodePortLocal feature is enabled, a port + # from that range will be assigned whenever a Pod's container defines a specific port to be exposed + # (each container can define a list of ports as pod.spec.containers[].ports), and all Node traffic + # directed to that port will be forwarded to the Pod. + portRange: "61000-62000" + + # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig. + # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver. + kubeAPIServerOverride: "" + + # Provide the address of DNS server, to override the kube-dns service. It's used to resolve hostname in FQDN policy. + # Defaults to "". It must be a host string or a host:port pair of the DNS server (e.g. 10.96.0.10, 10.96.0.10:53, + # [fd00:10:96::a]:53). + dnsServerOverride: "" + + # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. + # https://golang.org/pkg/crypto/tls/#pkg-constants + # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always + # prefer TLS1.3 Cipher Suites whenever possible. + tlsCipherSuites: "" + + # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. + tlsMinVersion: "" + + # The name of the interface on Node which is used for tunneling or routing the traffic across Nodes. + # If there are multiple IP addresses configured on the interface, the first one is used. The IP + # address used for tunneling or routing traffic to remote Nodes is decided in the following order of + # preference (from highest to lowest): + # 1. transportInterface + # 2. transportInterfaceCIDRs + # 3. The Node IP + transportInterface: "" + + multicast: + # To enable Multicast, you need to set "enable" to true, and ensure that the + # Multicast feature gate is also enabled (which is the default). + enable: false + + # The names of the interfaces on Nodes that are used to forward multicast traffic. + # Defaults to transport interface if not set. + multicastInterfaces: + + # The versions of IGMP queries antrea-agent sends to Pods. + # Valid versions are 1, 2 and 3. + igmpQueryVersions: + - 1 + - 2 + - 3 + + # The interval at which the antrea-agent sends IGMP queries to Pods. + # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + igmpQueryInterval: "10s" + + # The network CIDRs of the interface on Node which is used for tunneling or routing the traffic across + # Nodes. If there are multiple interfaces configured the same network CIDR, the first one is used. The + # IP address used for tunneling or routing traffic to remote Nodes is decided in the following order of + # preference (from highest to lowest): + # 1. transportInterface + # 2. transportInterfaceCIDRs + # 3. The Node IP + transportInterfaceCIDRs: + + # Option antreaProxy contains AntreaProxy related configuration options. + antreaProxy: + # ProxyAll tells antrea-agent to proxy all Service traffic, including NodePort, LoadBalancer, and ClusterIP traffic, + # regardless of where they come from. Therefore, running kube-proxy is no longer required. This requires the AntreaProxy + # feature to be enabled. + # Note that this option is experimental. If kube-proxy is removed, option kubeAPIServerOverride must be used to access + # apiserver directly. + proxyAll: false + # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks. + # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses. + # Note that the option is only valid when proxyAll is true. + nodePortAddresses: + # An array of string values to specify a list of Services which should be ignored by AntreaProxy (traffic to these + # Services will not be load-balanced). Values can be a valid ClusterIP (e.g. 10.11.1.2) or a Service name + # with Namespace (e.g. kube-system/kube-dns) + skipServices: + # When ProxyLoadBalancerIPs is set to false, AntreaProxy no longer load-balances traffic destined to the + # External IPs of LoadBalancer Services. This is useful when the external LoadBalancer provides additional + # capabilities (e.g. TLS termination) and it is desirable for Pod-to-ExternalIP traffic to be sent to the + # external LoadBalancer instead of being load-balanced to an Endpoint directly by AntreaProxy. + # Note that setting ProxyLoadBalancerIPs to false usually only makes sense when ProxyAll is set to true and + # kube-proxy is removed from the cluser, otherwise kube-proxy will still load-balance this traffic. + proxyLoadBalancerIPs: true + + # IPsec tunnel related configurations. + ipsec: + # The authentication mode of IPsec tunnel. It has the following options: + # - psk (default): Use pre-shared key (PSK) for IKE authentication. + # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` + # feature gate to be enabled. + authenticationMode: "psk" + + multicluster: + # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. + # This feature is supported only with encap mode. + enableGateway: false + # The Namespace where Antrea Multi-cluster Controller is running. + # The default is antrea-agent's Namespace. + namespace: "" + # Enable Multi-cluster NetworkPolicy (ingress rules). + # Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy. + enableStretchedNetworkPolicy: false + # Enable Pod to Pod connectivity. + enablePodToPodConnectivity: false + # Determines how cross-cluster traffic is encrypted. + # It has the following options: + # - none (default): Cross-cluster traffic will not be encrypted. + # - wireGuard: Use WireGuard to encrypt traffic. + trafficEncryptionMode: "none" + # WireGuard tunnel configuration for cross-cluster traffic. + # It only works when multicluster.trafficEncryptionMode is wireGuard. + wireGuard: + # WireGuard tunnel port for cross-cluster traffic. + port: 51821 + antrea-cni.conflist: | + { + "cniVersion":"0.3.0", + "name": "antrea", + "plugins": [ + { + "type": "antrea", + "ipam": { + "type": "host-local" + } + } + , + { + "type": "portmap", + "capabilities": {"portMappings": true} + } + , + { + "type": "bandwidth", + "capabilities": {"bandwidth": true} + } + ] + } + antrea-controller.conf: | + # FeatureGates is a map of feature names to bools that enable or disable experimental features. + featureGates: + # AllAlpha is a global toggle for alpha features. Per-feature key values override the default set by AllAlpha. + # AllAlpha: false + + # AllBeta is a global toggle for beta features. Per-feature key values override the default set by AllBeta. + # AllBeta: false + + # Enable traceflow which provides packet tracing feature to diagnose network issue. + # Traceflow: true + + # Enable Antrea ClusterNetworkPolicy feature to complement K8s NetworkPolicy for cluster admins + # to define security policies which apply to the entire cluster, and Antrea NetworkPolicy + # feature that supports priorities, rule actions and externalEntities in the future. + # AntreaPolicy: true + + # Enable collecting and exposing NetworkPolicy statistics. + # NetworkPolicyStats: true + + # Enable multicast traffic. + # Multicast: true + + # Enable controlling SNAT IPs of Pod egress traffic. + # Egress: true + + # Run Kubernetes NodeIPAMController with Antrea. + # NodeIPAM: true + + # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the + # bridging mode and allocates IPs to Pods in bridging mode. It is also required to use Antrea for + # IPAM when configuring secondary network interfaces with Multus. + # AntreaIPAM: false + + # Enable managing external IPs of Services of LoadBalancer type. + # ServiceExternalIP: false + + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + + # Enable managing ExternalNode for unmanaged VM/BM. + # ExternalNode: false + + # Enable collecting support bundle files with SupportBundleCollection CRD. + # SupportBundleCollection: false + + # Enable Antrea Multi-cluster features. + # Multicluster: false + + # Enable users to protect their applications by specifying how they are allowed to communicate with others, taking + # into account application context. + L7NetworkPolicy: true + + # The port for the antrea-controller APIServer to serve on. + # Note that if it's set to another value, the `containerPort` of the `api` port of the + # `antrea-controller` container must be set to the same value. + apiPort: 10349 + + # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener. + enablePrometheusMetrics: true + + # Indicates whether to use auto-generated self-signed TLS certificate. + # If false, a Secret named "antrea-controller-tls" must be provided with the following keys: + # ca.crt: + # tls.crt: + # tls.key: + selfSignedCert: true + + # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. + # https://golang.org/pkg/crypto/tls/#pkg-constants + # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always + # prefer TLS1.3 Cipher Suites whenever possible. + tlsCipherSuites: "" + + # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. + tlsMinVersion: "" + + # File path of the certificate bundle for all the signers that is recognized for incoming client + # certificates. + clientCAFile: "" + + nodeIPAM: + # Enable the integrated Node IPAM controller within the Antrea controller. + enableNodeIPAM: false + # CIDR ranges for Pods in cluster. String array containing single CIDR range, or multiple ranges. + # The CIDRs could be either IPv4 or IPv6. At most one CIDR may be specified for each IP family. + # Value ignored when enableNodeIPAM is false. + clusterCIDRs: + # CIDR ranges for Services in cluster. It is not necessary to specify it when there is no overlap with clusterCIDRs. + # Value ignored when enableNodeIPAM is false. + serviceCIDR: "" + serviceCIDRv6: "" + # Mask size for IPv4 Node CIDR in IPv4 or dual-stack cluster. Value ignored when enableNodeIPAM is false + # or when IPv4 Pod CIDR is not configured. Valid range is 16 to 30. + nodeCIDRMaskSizeIPv4: 24 + # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false + # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. + nodeCIDRMaskSizeIPv6: 64 + + ipsecCSRSigner: + # Determines the auto-approve policy of Antrea CSR signer for IPsec certificates management. + # If enabled, Antrea will auto-approve the CertificateSingingRequest (CSR) if its subject and x509 extensions + # are permitted, and the requestor can be validated. If K8s `BoundServiceAccountTokenVolume` feature is enabled, + # the Pod identity will also be validated to provide maximum security. + # If set to false, Antrea will not auto-approve CertificateSingingRequests and they need to be approved + # manually by `kubectl certificate approve`. + autoApprove: true + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # tls.crt: + # tls.key: + selfSignedCA: true + + multicluster: + # Enable Multi-cluster NetworkPolicy. + enableStretchedNetworkPolicy: false +--- +# Source: antrea/templates/crds/group.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: groups.crd.antrea.io +spec: + group: crd.antrea.io + versions: + - name: v1alpha3 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + childGroups: + type: array + items: + type: string + podSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + matchLabels: + x-kubernetes-preserve-unknown-fields: true + namespaceSelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + matchLabels: + x-kubernetes-preserve-unknown-fields: true + externalEntitySelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + matchLabels: + x-kubernetes-preserve-unknown-fields: true + ipBlocks: + type: array + items: + type: object + properties: + cidr: + type: string + format: cidr + serviceReference: + type: object + properties: + name: + type: string + namespace: + type: string + status: + type: object + properties: + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + lastTransitionTime: + type: string + subresources: + status: {} + scope: Namespaced + names: + plural: groups + singular: group + kind: Group + shortNames: + - grp +--- +# Source: antrea/templates/agent/clusterrole.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: antrea-agent + labels: + app: antrea +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - watch + - list + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list + - apiGroups: + - "" + resources: + - pods/status + verbs: + - patch + - apiGroups: + - "" + resources: + - endpoints + - services + - namespaces + verbs: + - get + - watch + - list + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - watch + - list + - apiGroups: + - crd.antrea.io + resources: + - antreaagentinfos + verbs: + - get + - update + - apiGroups: + - controlplane.antrea.io + resources: + - networkpolicies + - appliedtogroups + - addressgroups + verbs: + - get + - watch + - list + - apiGroups: + - controlplane.antrea.io + resources: + - egressgroups + verbs: + - get + - watch + - list + - apiGroups: + - controlplane.antrea.io + resources: + - nodestatssummaries + verbs: + - create + - apiGroups: + - controlplane.antrea.io + resources: + - networkpolicies/status + verbs: + - create + - get + - apiGroups: + - controlplane.antrea.io + resources: + - supportbundlecollections + verbs: + - get + - watch + - list + - apiGroups: + - controlplane.antrea.io + resources: + - supportbundlecollections/status + verbs: + - create + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create + # This is the content of built-in role kube-system/extension-apiserver-authentication-reader. + # But it doesn't have list/watch permission before K8s v1.17.0 so the extension apiserver (antrea-agent) will + # have permission issue after bumping up apiserver library to a version that supports dynamic authentication. + # See https://github.com/kubernetes/kubernetes/pull/85375 + # To support K8s clusters older than v1.17.0, we grant the required permissions directly instead of relying on + # the extension-apiserver-authentication role. + - apiGroups: + - "" + resourceNames: + - extension-apiserver-authentication + resources: + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - antrea-ca + verbs: + - get + - watch + - list + - apiGroups: + - crd.antrea.io + resources: + - traceflows + - traceflows/status + verbs: + - get + - watch + - list + - update + - patch + - create + - delete + - apiGroups: + - crd.antrea.io + resources: + - egresses + verbs: + - get + - watch + - list + - apiGroups: + - crd.antrea.io + resources: + - egresses/status + verbs: + - update + - apiGroups: + - crd.antrea.io + resources: + - externalippools + - ippools + verbs: + - get + - watch + - list + - apiGroups: + - crd.antrea.io + resources: + - ippools/status + verbs: + - update + - apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - watch + - list + - create + - apiGroups: + - multicluster.crd.antrea.io + resources: + - gateways + verbs: + - get + - list + - watch + - patch + - apiGroups: + - multicluster.crd.antrea.io + resources: + - clusterinfoimports + - labelidentities + verbs: + - get + - list + - watch + - apiGroups: + - crd.antrea.io + resources: + - trafficcontrols + verbs: + - get + - watch + - list + - update + - patch + - create + - delete +--- +# Source: antrea/templates/antctl/clusterrole.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: antctl + labels: + app: antrea +rules: + - apiGroups: + - controlplane.antrea.io + resources: + - networkpolicies + - appliedtogroups + - addressgroups + verbs: + - get + - list + - apiGroups: + - stats.antrea.io + resources: + - networkpolicystats + - antreaclusternetworkpolicystats + - antreanetworkpolicystats + verbs: + - get + - list + - apiGroups: + - system.antrea.io + resources: + - controllerinfos + - agentinfos + verbs: + - get + - apiGroups: + - system.antrea.io + resources: + - supportbundles + verbs: + - get + - post + - apiGroups: + - system.antrea.io + resources: + - supportbundles/download + verbs: + - get + - nonResourceURLs: + - /agentinfo + - /addressgroups + - /appliedtogroups + - /loglevel + - /networkpolicies + - /ovsflows + - /ovstracing + - /podinterfaces + - /featuregates + - /serviceexternalip + verbs: + - get +--- +# Source: antrea/templates/cluster-identity-reader/clusterrolebinding.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: antrea-cluster-identity-reader + labels: + app: antrea +rules: + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - antrea-cluster-identity + verbs: + - get +--- +# Source: antrea/templates/controller/clusterrole.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: antrea-controller + labels: + app: antrea +rules: + - apiGroups: + - "" + resources: + - pods + - services + - namespaces + - configmaps + verbs: + - get + - watch + - list + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - services/status + verbs: + - update + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - watch + - list + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - update + # This is the content of built-in role kube-system/extension-apiserver-authentication-reader. + # But it doesn't have list/watch permission before K8s v1.17.0 so the extension apiserver (antrea-controller) will + # have permission issue after bumping up apiserver library to a version that supports dynamic authentication. + # See https://github.com/kubernetes/kubernetes/pull/85375 + # To support K8s clusters older than v1.17.0, we grant the required permissions directly instead of relying on + # the extension-apiserver-authentication role. + - apiGroups: + - "" + resourceNames: + - extension-apiserver-authentication + resources: + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - antrea-ca + - antrea-ipsec-ca + - antrea-cluster-identity + verbs: + - get + - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-ipsec-ca + verbs: + - get + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - create + - apiGroups: + - apiregistration.k8s.io + resources: + - apiservices + verbs: + - list + - update + - apiGroups: + - apiregistration.k8s.io + resources: + - apiservices + resourceNames: + - v1beta1.networking.antrea.tanzu.vmware.com + - v1beta1.controlplane.antrea.tanzu.vmware.com + - v1alpha1.stats.antrea.tanzu.vmware.com + - v1beta1.system.antrea.tanzu.vmware.com + - v1beta2.controlplane.antrea.tanzu.vmware.com + verbs: + - delete + - apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - list + - update + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - antrea.io/antrea-agent-ipsec-tunnel + verbs: + - approve + - sign + - apiGroups: + - crd.antrea.io + resources: + - antreacontrollerinfos + verbs: + - get + - create + - update + - delete + - apiGroups: + - crd.antrea.io + resources: + - antreaagentinfos + verbs: + - list + - create + - delete + - apiGroups: + - crd.antrea.io + resources: + - clusternetworkpolicies + - networkpolicies + verbs: + - get + - watch + - list + - update + - patch + - create + - delete + - apiGroups: + - crd.antrea.io + resources: + - clusternetworkpolicies/status + - networkpolicies/status + verbs: + - update + - apiGroups: + - crd.antrea.io + resources: + - tiers + verbs: + - get + - watch + - list + - update + - patch + - create + - delete + - apiGroups: + - crd.antrea.io + resources: + - traceflows + - traceflows/status + verbs: + - get + - watch + - list + - update + - patch + - create + - delete + - apiGroups: + - crd.antrea.io + resources: + - externalentities + - clustergroups + - groups + verbs: + - get + - watch + - list + - update + - patch + - create + - delete + - apiGroups: + - crd.antrea.io + resources: + - clustergroups/status + - groups/status + verbs: + - update + - apiGroups: + - crd.antrea.io + resources: + - egresses + verbs: + - get + - watch + - list + - update + - patch + - apiGroups: + - crd.antrea.io + resources: + - externalippools + - ippools + verbs: + - get + - watch + - list + - apiGroups: + - crd.antrea.io + resources: + - externalippools/status + - ippools/status + verbs: + - update + - patch + - apiGroups: + - crd.antrea.io + resources: + - externalnodes + verbs: + - get + - watch + - list + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - watch + - apiGroups: + - crd.antrea.io + resources: + - supportbundlecollections + verbs: + - get + - watch + - list + - apiGroups: + - crd.antrea.io + resources: + - supportbundlecollections/status + verbs: + - update + - apiGroups: + - multicluster.crd.antrea.io + resources: + - labelidentities + verbs: + - get + - list + - watch +--- +# Source: antrea/templates/crds-rbac/clusterroles.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: aggregate-antrea-policies-edit + labels: + app: antrea + # Add these permissions to the "admin" and "edit" default roles. + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" +rules: +- apiGroups: ["crd.antrea.io"] + resources: ["clusternetworkpolicies", "networkpolicies"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +# Source: antrea/templates/crds-rbac/clusterroles.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: aggregate-antrea-policies-view + labels: + app: antrea + # Add these permissions to the "view" default role. + rbac.authorization.k8s.io/aggregate-to-view: "true" +rules: +- apiGroups: ["crd.antrea.io"] + resources: ["clusternetworkpolicies", "networkpolicies"] + verbs: ["get", "list", "watch"] +--- +# Source: antrea/templates/crds-rbac/clusterroles.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: aggregate-traceflows-edit + labels: + app: antrea + # Add these permissions to the "admin" and "edit" default roles. + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" +rules: +- apiGroups: ["crd.antrea.io"] + resources: ["traceflows"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +# Source: antrea/templates/crds-rbac/clusterroles.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: aggregate-traceflows-view + labels: + app: antrea + # Add these permissions to the "view" default role. + rbac.authorization.k8s.io/aggregate-to-view: "true" +rules: +- apiGroups: ["crd.antrea.io"] + resources: ["traceflows"] + verbs: ["get", "list", "watch"] +--- +# Source: antrea/templates/crds-rbac/clusterroles.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: aggregate-antrea-clustergroups-edit + labels: + app: antrea + # Add these permissions to the "admin" and "edit" default roles. + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" +rules: +- apiGroups: ["crd.antrea.io"] + resources: ["clustergroups"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +# Source: antrea/templates/crds-rbac/clusterroles.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: aggregate-antrea-clustergroups-view + labels: + app: antrea + # Add these permissions to the "view" default role. + rbac.authorization.k8s.io/aggregate-to-view: "true" +rules: +- apiGroups: ["crd.antrea.io"] + resources: ["clustergroups"] + verbs: ["get", "list", "watch"] +--- +# Source: antrea/templates/agent/clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: antrea-agent + labels: + app: antrea +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: antrea-agent +subjects: + - kind: ServiceAccount + name: antrea-agent + namespace: kube-system +--- +# Source: antrea/templates/antctl/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: antrea + name: antctl +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: antctl +subjects: + - kind: ServiceAccount + name: antctl + namespace: kube-system +--- +# Source: antrea/templates/controller/clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: antrea-controller + labels: + app: antrea +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: antrea-controller +subjects: + - kind: ServiceAccount + name: antrea-controller + namespace: kube-system +--- +# Source: antrea/templates/controller/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: antrea + namespace: kube-system + labels: + app: antrea +spec: + ports: + - port: 443 + protocol: TCP + targetPort: api + selector: + app: antrea + component: antrea-controller +--- +# Source: antrea/templates/agent/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: antrea-agent + namespace: kube-system + labels: + app: antrea + component: antrea-agent +spec: + selector: + matchLabels: + app: antrea + component: antrea-agent + updateStrategy: + type: RollingUpdate + template: + metadata: + annotations: + # Starting with v1.21, Kubernetes supports default container annotation. + # Using "kubectl logs/exec/attach/cp" doesn't have to specify "-c antrea-agent" when troubleshooting. + kubectl.kubernetes.io/default-container: antrea-agent + # Automatically restart Pods with a RollingUpdate if the ConfigMap changes + # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments + checksum/config: 580bdef7b0694c245b3ec7a8ae1d58d25dc6d5c6065246f91d9e74a7ddad9d91 + labels: + app: antrea + component: antrea-agent + spec: + hostNetwork: true + priorityClassName: system-node-critical + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: antrea-agent + initContainers: + - name: install-cni + image: "antrea/antrea-ubuntu:latest" + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + command: ["install_cni"] + securityContext: + capabilities: + add: + # SYS_MODULE is required to load the OVS kernel module. + - SYS_MODULE + env: + # SKIP_CNI_BINARIES takes in values as a comma separated list of + # binaries that need to be skipped for installation, e.g. "portmap, bandwidth". + - name: SKIP_CNI_BINARIES + value: "" + volumeMounts: + - name: antrea-config + mountPath: /etc/antrea/antrea-cni.conflist + subPath: antrea-cni.conflist + readOnly: true + - name: host-cni-conf + mountPath: /host/etc/cni/net.d + - name: host-cni-bin + mountPath: /host/opt/cni/bin + # For loading the OVS kernel module. + - name: host-lib-modules + mountPath: /lib/modules + readOnly: true + # For changing the default permissions of the run directory. + - name: host-var-run-antrea + mountPath: /var/run/antrea + containers: + - name: antrea-agent + image: "antrea/antrea-ubuntu:latest" + imagePullPolicy: IfNotPresent + command: ["antrea-agent"] + # Log to both "/var/log/antrea/" and stderr (so "kubectl logs" can work).- + args: + - "--config=/etc/antrea/antrea-agent.conf" + - "--logtostderr=false" + - "--log_dir=/var/log/antrea" + - "--alsologtostderr" + - "--log_file_max_size=100" + - "--log_file_max_num=4" + env: + # Provide pod and node information for clusterinformation CRD. + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 200m + ports: + - containerPort: 10350 + name: api + protocol: TCP + livenessProbe: + httpGet: + host: localhost + path: /livez + port: api + scheme: HTTPS + initialDelaySeconds: 10 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 5 + readinessProbe: + httpGet: + host: localhost + path: /readyz + port: api + scheme: HTTPS + initialDelaySeconds: 5 + timeoutSeconds: 5 + periodSeconds: 10 + # In large-scale clusters, it may take up to 40~50 seconds for the antrea-agent to reconnect to the antrea + # Service after the antrea-controller restarts. The antrea-agent shouldn't be reported as NotReady in this + # scenario, otherwise the DaemonSet controller would restart all agents at once, as opposed to performing a + # rolling update. Set failureThreshold to 8 so it can tolerate 70s of disconnection. + failureThreshold: 8 + securityContext: + # antrea-agent needs to perform sysctl configuration. + privileged: true + volumeMounts: + - name: antrea-config + mountPath: /etc/antrea/antrea-agent.conf + subPath: antrea-agent.conf + readOnly: true + - name: host-var-run-antrea + mountPath: /var/run/antrea + - name: host-var-run-antrea + mountPath: /var/run/openvswitch + subPath: openvswitch + # host-local IPAM stores allocated IP addresses as files in /var/lib/cni/networks/$NETWORK_NAME. + # Mount a sub-directory of host-var-run-antrea to it for persistence of IP allocation. + - name: host-var-run-antrea + mountPath: /var/lib/cni + subPath: cni + # We need to mount both the /proc directory and the /var/run/netns directory so that + # antrea-agent can open the network namespace path when setting up Pod + # networking. Different container runtimes may use /proc or /var/run/netns when invoking + # the CNI commands. Docker uses /proc and containerd uses /var/run/netns. + - name: host-var-log-antrea + mountPath: /var/log/antrea + - name: host-proc + mountPath: /host/proc + readOnly: true + - name: host-var-run-netns + mountPath: /host/var/run/netns + readOnly: true + # When a container is created, a mount point for the network namespace is added under + # /var/run/netns on the host, which needs to be propagated to the antrea-agent container. + mountPropagation: HostToContainer + - name: xtables-lock + mountPath: /run/xtables.lock + - name: antrea-ovs + image: "antrea/antrea-ubuntu:latest" + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 200m + command: ["start_ovs"] + args: + - "--log_file_max_size=100" + - "--log_file_max_num=4" + securityContext: + # capabilities required by OVS daemons + capabilities: + add: + - SYS_NICE + - NET_ADMIN + - SYS_ADMIN + - IPC_LOCK + livenessProbe: + exec: + # docker CRI doesn't honor timeoutSeconds, add "timeout" to the command as a workaround. + # https://github.com/kubernetes/kubernetes/issues/51901 + command: + - /bin/sh + - -c + - timeout 10 container_liveness_probe ovs + initialDelaySeconds: 5 + timeoutSeconds: 10 + periodSeconds: 10 + failureThreshold: 5 + volumeMounts: + - name: host-var-run-antrea + mountPath: /var/run/openvswitch + subPath: openvswitch + - name: host-var-log-antrea + mountPath: /var/log/openvswitch + subPath: openvswitch + volumes: + - name: antrea-config + configMap: + name: antrea-config + - name: host-cni-conf + hostPath: + path: /etc/cni/net.d + - name: host-cni-bin + hostPath: + path: /opt/cni/bin + - name: host-proc + hostPath: + path: /proc + - name: host-var-run-netns + hostPath: + path: /var/run/netns + - name: host-var-run-antrea + hostPath: + path: /var/run/antrea + # we use subPath to create run subdirectories for different component (e.g. OVS) and + # subPath requires the base volume to exist + type: DirectoryOrCreate + - name: host-var-log-antrea + hostPath: + path: /var/log/antrea + # we use subPath to create logging subdirectories for different component (e.g. OVS) + type: DirectoryOrCreate + - name: host-lib-modules + hostPath: + path: /lib/modules + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate +--- +# Source: antrea/templates/controller/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: antrea-controller + namespace: kube-system + labels: + app: antrea + component: antrea-controller +spec: + strategy: + # Ensure the existing Pod is stopped before the new one is created. + type: Recreate + selector: + matchLabels: + app: antrea + component: antrea-controller + replicas: 1 + template: + metadata: + annotations: + # Automatically restart Pod if the ConfigMap changes + # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments + checksum/config: 580bdef7b0694c245b3ec7a8ae1d58d25dc6d5c6065246f91d9e74a7ddad9d91 + labels: + app: antrea + component: antrea-controller + spec: + nodeSelector: + kubernetes.io/os: linux + hostNetwork: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + serviceAccountName: antrea-controller + containers: + - name: antrea-controller + image: "antrea/antrea-ubuntu:latest" + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 200m + command: ["antrea-controller"] + # Log to both "/var/log/antrea/" and stderr (so "kubectl logs" can work). + args: + - "--config=/etc/antrea/antrea-controller.conf" + - "--logtostderr=false" + - "--log_dir=/var/log/antrea" + - "--alsologtostderr" + - "--log_file_max_size=100" + - "--log_file_max_num=4" + env: + # Provide pod and node information for clusterinformation CRD. + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Provide ServiceAccount name for validation webhook. + - name: SERVICEACCOUNT_NAME + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: ANTREA_CONFIG_MAP_NAME + value: antrea-config + ports: + - containerPort: 10349 + name: api + protocol: TCP + readinessProbe: + httpGet: + host: localhost + path: /readyz + port: api + scheme: HTTPS + initialDelaySeconds: 5 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 5 + livenessProbe: + httpGet: + host: localhost + path: /livez + port: api + scheme: HTTPS + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 5 + volumeMounts: + - name: antrea-config + mountPath: /etc/antrea/antrea-controller.conf + subPath: antrea-controller.conf + readOnly: true + - name: antrea-controller-tls + mountPath: /var/run/antrea/antrea-controller-tls + - name: host-var-log-antrea + mountPath: /var/log/antrea + volumes: + - name: antrea-config + configMap: + name: antrea-config + # Make it optional as we only read it when selfSignedCert=false. + - name: antrea-controller-tls + secret: + secretName: antrea-controller-tls + defaultMode: 0400 + optional: true + - name: host-var-log-antrea + hostPath: + path: /var/log/antrea + type: DirectoryOrCreate +--- +# Source: antrea/templates/controller/apiservices.yaml +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta2.controlplane.antrea.io + labels: + app: antrea + served-by: antrea-controller +spec: + group: controlplane.antrea.io + groupPriorityMinimum: 100 + version: v1beta2 + versionPriority: 100 + service: + name: antrea + namespace: kube-system +--- +# Source: antrea/templates/controller/apiservices.yaml +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.system.antrea.io + labels: + app: antrea + served-by: antrea-controller +spec: + group: system.antrea.io + groupPriorityMinimum: 100 + version: v1beta1 + versionPriority: 100 + service: + name: antrea + namespace: kube-system +--- +# Source: antrea/templates/controller/apiservices.yaml +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1alpha1.stats.antrea.io + labels: + app: antrea + served-by: antrea-controller +spec: + group: stats.antrea.io + groupPriorityMinimum: 100 + version: v1alpha1 + versionPriority: 100 + service: + name: antrea + namespace: kube-system +--- +# Source: antrea/templates/webhooks/mutating/crdmutator.yaml +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: "crdmutator.antrea.io" + labels: + app: antrea + served-by: antrea-controller +webhooks: + - name: "acnpmutator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: kube-system + path: "/mutate/acnp" + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha1"] + resources: ["clusternetworkpolicies"] + scope: "Cluster" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 + - name: "anpmutator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: kube-system + path: "/mutate/anp" + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha1"] + resources: ["networkpolicies"] + scope: "Namespaced" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 +--- +# Source: antrea/templates/webhooks/validating/crdvalidator.yaml +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: "crdvalidator.antrea.io" + labels: + app: antrea + served-by: antrea-controller +webhooks: + - name: "tiervalidator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: kube-system + path: "/validate/tier" + rules: + - operations: ["CREATE", "UPDATE", "DELETE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha1"] + resources: ["tiers"] + scope: "Cluster" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 + - name: "acnpvalidator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: kube-system + path: "/validate/acnp" + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha1"] + resources: ["clusternetworkpolicies"] + scope: "Cluster" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 + - name: "anpvalidator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: kube-system + path: "/validate/anp" + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha1"] + resources: ["networkpolicies"] + scope: "Namespaced" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 + - name: "clustergroupvalidator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: kube-system + path: "/validate/clustergroup" + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha3", "v1alpha2"] + resources: ["clustergroups"] + scope: "Cluster" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 + - name: "groupvalidator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: "kube-system" + path: "/validate/group" + rules: + - operations: [ "CREATE", "UPDATE" ] + apiGroups: [ "crd.antrea.io" ] + apiVersions: [ "v1alpha3" ] + resources: [ "groups" ] + scope: "Namespaced" + admissionReviewVersions: [ "v1", "v1beta1" ] + sideEffects: None + timeoutSeconds: 5 + - name: "externalippoolvalidator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: kube-system + path: "/validate/externalippool" + rules: + - operations: ["UPDATE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha2"] + resources: ["externalippools"] + scope: "Cluster" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 + - name: "egressvalidator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: kube-system + path: "/validate/egress" + rules: + - operations: ["CREATE", "UPDATE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha2"] + resources: ["egresses"] + scope: "Cluster" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 + - name: "ippoolvalidator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: kube-system + path: "/validate/ippool" + rules: + - operations: ["CREATE", "UPDATE", "DELETE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha2"] + resources: ["ippools"] + scope: "Cluster" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 + - name: "supportbundlecollectionvalidator.antrea.io" + clientConfig: + service: + name: "antrea" + namespace: kube-system + path: "/validate/supportbundlecollection" + rules: + - operations: ["UPDATE", "DELETE"] + apiGroups: ["crd.antrea.io"] + apiVersions: ["v1alpha1"] + resources: ["supportbundlecollections"] + scope: "Cluster" + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None + timeoutSeconds: 5 diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf index dc0a7faea3c..de8a4d639a6 100644 --- a/build/charts/antrea/conf/antrea-agent.conf +++ b/build/charts/antrea/conf/antrea-agent.conf @@ -73,6 +73,9 @@ featureGates: # into account application context. {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "L7NetworkPolicy" "default" false) }} +# Enable L7Visibility on Pods and Namespace. +{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "L7Visibility" "default" false) }} + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: {{ .Values.ovs.bridgeName | quote }} diff --git a/build/charts/antrea/templates/agent/clusterrole.yaml b/build/charts/antrea/templates/agent/clusterrole.yaml index 7db11aebb8e..aaf95e3725a 100644 --- a/build/charts/antrea/templates/agent/clusterrole.yaml +++ b/build/charts/antrea/templates/agent/clusterrole.yaml @@ -173,7 +173,6 @@ rules: resources: - externalippools - ippools - - trafficcontrols verbs: - get - watch @@ -219,3 +218,15 @@ rules: - get - list - watch + - apiGroups: + - crd.antrea.io + resources: + - trafficcontrols + verbs: + - get + - watch + - list + - update + - patch + - create + - delete diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml index 76d01ec2568..49cd73b70db 100644 --- a/build/charts/antrea/values.yaml +++ b/build/charts/antrea/values.yaml @@ -300,7 +300,7 @@ controller: flowExporter: # -- Enable the flow exporter feature. - enable: false + enable: true # -- IPFIX collector address as a string with format :[][:]. # If the collector is running in-cluster as a Service, set to # /. diff --git a/build/charts/flow-aggregator/values.yaml b/build/charts/flow-aggregator/values.yaml index e88b9ea321a..e7f4e2774e7 100644 --- a/build/charts/flow-aggregator/values.yaml +++ b/build/charts/flow-aggregator/values.yaml @@ -44,7 +44,7 @@ flowCollector: # clickHouse contains ClickHouse related configuration options. clickHouse: # -- Determine whether to enable exporting flow records to ClickHouse. - enable: false + enable: true # -- DatabaseURL is the url to the database. TCP protocol is required. databaseURL: "tcp://clickhouse-clickhouse.flow-visibility.svc:9000" # -- Debug enables debug logs from ClickHouse sql driver. diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index e8b2d22073e..4e4f58685dd 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -3165,7 +3165,7 @@ data: # IPFIX flow records from each agent to a configured collector. To enable this # feature, you need to set "enable" to true, and ensure that the FlowExporter # feature gate is also enabled. - enable: false + enable: true # Provide the IPFIX collector address as a string with format :[][:]. # HOST can either be the DNS name, IP, or Service name of the Flow Collector. If # using an IP, it can be either IPv4 or IPv6. However, IPv6 address should be @@ -4371,7 +4371,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: abf7cb1c21b730664510e8a762d967df5c620467f12bf3e0bae41df73489de65 + checksum/config: 0d24b57f9f10487ac37067c795ca8149ea5e775935494afd77f0321b17f0ee3e labels: app: antrea component: antrea-agent @@ -4612,7 +4612,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: abf7cb1c21b730664510e8a762d967df5c620467f12bf3e0bae41df73489de65 + checksum/config: 0d24b57f9f10487ac37067c795ca8149ea5e775935494afd77f0321b17f0ee3e labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 116ebf97b44..cd0179fb921 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -3165,7 +3165,7 @@ data: # IPFIX flow records from each agent to a configured collector. To enable this # feature, you need to set "enable" to true, and ensure that the FlowExporter # feature gate is also enabled. - enable: false + enable: true # Provide the IPFIX collector address as a string with format :[][:]. # HOST can either be the DNS name, IP, or Service name of the Flow Collector. If # using an IP, it can be either IPv4 or IPv6. However, IPv6 address should be @@ -4371,7 +4371,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: abf7cb1c21b730664510e8a762d967df5c620467f12bf3e0bae41df73489de65 + checksum/config: 0d24b57f9f10487ac37067c795ca8149ea5e775935494afd77f0321b17f0ee3e labels: app: antrea component: antrea-agent @@ -4613,7 +4613,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: abf7cb1c21b730664510e8a762d967df5c620467f12bf3e0bae41df73489de65 + checksum/config: 0d24b57f9f10487ac37067c795ca8149ea5e775935494afd77f0321b17f0ee3e labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 68470ecebeb..c284e6b2517 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -3165,7 +3165,7 @@ data: # IPFIX flow records from each agent to a configured collector. To enable this # feature, you need to set "enable" to true, and ensure that the FlowExporter # feature gate is also enabled. - enable: false + enable: true # Provide the IPFIX collector address as a string with format :[][:]. # HOST can either be the DNS name, IP, or Service name of the Flow Collector. If # using an IP, it can be either IPv4 or IPv6. However, IPv6 address should be @@ -4371,7 +4371,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: e119a0592b2cef130a915258e9f23a8e4ed0aa28685841dc6f0d4b72a6983beb + checksum/config: 41a1c1ed5706524ad0bb2655ebf792087177625e303281e3658246a701eca329 labels: app: antrea component: antrea-agent @@ -4610,7 +4610,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: e119a0592b2cef130a915258e9f23a8e4ed0aa28685841dc6f0d4b72a6983beb + checksum/config: 41a1c1ed5706524ad0bb2655ebf792087177625e303281e3658246a701eca329 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 72b5ae5ba17..c9619596f13 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -3178,7 +3178,7 @@ data: # IPFIX flow records from each agent to a configured collector. To enable this # feature, you need to set "enable" to true, and ensure that the FlowExporter # feature gate is also enabled. - enable: false + enable: true # Provide the IPFIX collector address as a string with format :[][:]. # HOST can either be the DNS name, IP, or Service name of the Flow Collector. If # using an IP, it can be either IPv4 or IPv6. However, IPv6 address should be @@ -4384,7 +4384,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 373bbb8d6a42a8f3e546ea446fd078c04e947093e68e0c1c973f4c696bf8d607 + checksum/config: eb5f06767bd483472a70515a7055416d9904e0c08c4fa8563b5955937008750e checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4 labels: app: antrea @@ -4669,7 +4669,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 373bbb8d6a42a8f3e546ea446fd078c04e947093e68e0c1c973f4c696bf8d607 + checksum/config: eb5f06767bd483472a70515a7055416d9904e0c08c4fa8563b5955937008750e labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index f3f18f3a3a3..524d6abe15d 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -3165,7 +3165,7 @@ data: # IPFIX flow records from each agent to a configured collector. To enable this # feature, you need to set "enable" to true, and ensure that the FlowExporter # feature gate is also enabled. - enable: false + enable: true # Provide the IPFIX collector address as a string with format :[][:]. # HOST can either be the DNS name, IP, or Service name of the Flow Collector. If # using an IP, it can be either IPv4 or IPv6. However, IPv6 address should be @@ -4371,7 +4371,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 7e1d1a60a99fdbe25ff10b92e85ff234a5769c02bd9ae0ead56a3fe0a8ad118d + checksum/config: 4b484dbf100647570cc3ab0bacb16d2207918c8258d58d24f172efe14b62512f labels: app: antrea component: antrea-agent @@ -4610,7 +4610,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 7e1d1a60a99fdbe25ff10b92e85ff234a5769c02bd9ae0ead56a3fe0a8ad118d + checksum/config: 4b484dbf100647570cc3ab0bacb16d2207918c8258d58d24f172efe14b62512f labels: app: antrea component: antrea-controller diff --git a/build/yamls/flow-aggregator.yml b/build/yamls/flow-aggregator.yml index 09d1e690ed5..545a3b89786 100644 --- a/build/yamls/flow-aggregator.yml +++ b/build/yamls/flow-aggregator.yml @@ -219,7 +219,7 @@ data: # clickHouse contains ClickHouse related configuration options. clickHouse: # Enable is the switch to enable exporting flow records to ClickHouse. - enable: false + enable: true # Database is the name of database where Antrea "flows" table is created. database: "default" diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index bfcd250ca33..eada99f7f75 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -39,6 +39,7 @@ import ( "antrea.io/antrea/pkg/agent/controller/egress" "antrea.io/antrea/pkg/agent/controller/ipseccertificate" "antrea.io/antrea/pkg/agent/controller/networkpolicy" + "antrea.io/antrea/pkg/agent/controller/networkpolicy/l7engine" "antrea.io/antrea/pkg/agent/controller/noderoute" "antrea.io/antrea/pkg/agent/controller/serviceexternalip" "antrea.io/antrea/pkg/agent/controller/traceflow" @@ -691,6 +692,7 @@ func run(o *Options) error { if features.DefaultFeatureGate.Enabled(features.TrafficControl) { tcController := trafficcontrol.NewTrafficControlController(ofClient, + crdClient, ifaceStore, ovsBridgeClient, ovsCtlClient, @@ -699,6 +701,9 @@ func run(o *Options) error { namespaceInformer, podUpdateChannel) go tcController.Run(stopCh) + if features.DefaultFeatureGate.Enabled(features.L7Visibility) { + go l7engine.Run(tcController) + } } // Start the localPodInformer diff --git a/go.mod b/go.mod index 5c79043db89..ae44d245b2c 100644 --- a/go.mod +++ b/go.mod @@ -207,3 +207,5 @@ require ( sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) + +replace github.com/vmware/go-ipfix v0.6.1 => github.com/tushartathgur/go-ipfix v0.0.0-20230714214305-c3c7a13353f8 diff --git a/go.sum b/go.sum index 68f60a98337..af288533338 100644 --- a/go.sum +++ b/go.sum @@ -1047,6 +1047,8 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= +github.com/tushartathgur/go-ipfix v0.0.0-20230714214305-c3c7a13353f8 h1:5f0hm5NzcSY4tKiKInkCGLnps+DDWEVK5qV9wYnE9/g= +github.com/tushartathgur/go-ipfix v0.0.0-20230714214305-c3c7a13353f8/go.mod h1:dGCppoeqknr9o3yz9BD74mP/FPHgefb6v34xdUKxDPI= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -1064,8 +1066,6 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vmware/go-ipfix v0.6.1 h1:6Gh4kuUkPwetEJ6dWxlz029qwgFBXtcYYVWuiqy0Zgo= -github.com/vmware/go-ipfix v0.6.1/go.mod h1:dGCppoeqknr9o3yz9BD74mP/FPHgefb6v34xdUKxDPI= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= diff --git a/pkg/agent/controller/networkpolicy/l7engine/l7visbilitycontroller.go b/pkg/agent/controller/networkpolicy/l7engine/l7visbilitycontroller.go new file mode 100644 index 00000000000..d8edcd61c56 --- /dev/null +++ b/pkg/agent/controller/networkpolicy/l7engine/l7visbilitycontroller.go @@ -0,0 +1,264 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package l7engine + +import ( + "context" + "os" + "os/exec" + "strings" + "sync" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/apis/crd/v1alpha2" + crdclientset "antrea.io/antrea/pkg/client/clientset/versioned" + + "antrea.io/antrea/pkg/agent/controller/trafficcontrol" +) + +var ( + L7mutex sync.Mutex + crdClient crdclientset.Interface +) + +func Run(tcController *trafficcontrol.Controller) { + // Set up kubeconfig flag + kubeconfig, err := ResolveKubeConfig() + if err != nil { + klog.ErrorS(err, "") + } + + clientset, err := kubernetes.NewForConfig(kubeconfig) + if err != nil { + klog.ErrorS(err, "") + } + + // Create a watcher for pods and namespaces + podWatcher, err := clientset.CoreV1().Pods("").Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: fields.Everything().String(), + }) + if err != nil { + klog.ErrorS(err, "") + } + defer podWatcher.Stop() + + namespaceWatcher, err := clientset.CoreV1().Namespaces().Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: fields.Everything().String(), + }) + if err != nil { + klog.ErrorS(err, "") + } + defer namespaceWatcher.Stop() + + // Set up channels to receive watch events + podEvents := podWatcher.ResultChan() + namespaceEvents := namespaceWatcher.ResultChan() + l7Reconciler := NewReconciler() + + // Start listening for events + go processPodEvents(clientset, podEvents, l7Reconciler, tcController) + go processNamespaceEvents(clientset, namespaceEvents, l7Reconciler, tcController) + + // Keep the program running indefinitely + select {} +} + +func processPodEvents(clientset *kubernetes.Clientset, events <-chan watch.Event, l7Reconciler *Reconciler, tcController *trafficcontrol.Controller) { + direction := v1alpha2.DirectionBoth + for event := range events { + pod, ok := event.Object.(*corev1.Pod) + if !ok { + klog.InfoS("Received unexpected object: ", "", event.Object) + continue + } + if event.Type == watch.Added || event.Type == watch.Modified { + for key, value := range pod.Annotations { + if key == "l7visibility" { + elements := strings.Split(value, "/") + switch elements[0] { + case "Ingress": + direction = v1alpha2.DirectionIngress + case "Egress": + direction = v1alpha2.DirectionEgress + case "Both": + direction = v1alpha2.DirectionBoth + } + nodeName := pod.Spec.NodeName + podLabels := pod.ObjectMeta.Labels + // Get the namespace of the pod + namespaceName := pod.ObjectMeta.Namespace + // Get the labels of the namespace + namespaceObj, err := clientset.CoreV1().Namespaces().Get(context.TODO(), namespaceName, metav1.GetOptions{}) + if err != nil { + klog.ErrorS(err, "") + return + } + namespaceLabels := namespaceObj.ObjectMeta.Labels + + StartSuricataIfNotRunning(clientset, nodeName, l7Reconciler) + postTrafficControl(direction, podLabels, namespaceLabels, tcController) + } + } + } else if event.Type == watch.Deleted { + for key, _ := range pod.Annotations { + if key == "l7-visibility" { + continue + } + } + } + } +} + +func processNamespaceEvents(clientset *kubernetes.Clientset, events <-chan watch.Event, l7Reconciler *Reconciler, tcController *trafficcontrol.Controller) { + direction := v1alpha2.DirectionBoth + nodeList, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + FieldSelector: fields.Everything().String(), + LabelSelector: labels.Everything().String(), + }) + if err != nil { + klog.Errorf("Failed to list all nodes: %v", err) + } + for event := range events { + namespace, ok := event.Object.(*corev1.Namespace) + if !ok { + klog.InfoS("Received unexpected object: ", "", event.Object) + continue + } + if event.Type == watch.Added || event.Type == watch.Modified { + for key, value := range namespace.Annotations { + if key == "l7-visibility" { + elements := strings.Split(value, "/") + switch elements[0] { + case "Ingress": + direction = v1alpha2.DirectionIngress + case "Egress": + direction = v1alpha2.DirectionEgress + case "Both": + direction = v1alpha2.DirectionBoth + } + namespaceLabels := namespace.ObjectMeta.Labels + for _, node := range nodeList.Items { + StartSuricataIfNotRunning(clientset, node.Name, l7Reconciler) + postTrafficControl(direction, nil, namespaceLabels, tcController) + } + } + } + } else if event.Type == watch.Deleted { + for key, _ := range namespace.Annotations { + if key == "l7-visibility" { + continue + } + } + } + } +} + +func ResolveKubeConfig() (*rest.Config, error) { + var hasIt bool + var kubeconfigPath string + var err error + kubeconfigPath, hasIt = os.LookupEnv("KUBECONFIG") + if !hasIt || len(strings.TrimSpace(kubeconfigPath)) == 0 { + kubeconfigPath = clientcmd.RecommendedHomeFile + } + if _, err = os.Stat(kubeconfigPath); kubeconfigPath == clientcmd.RecommendedHomeFile && os.IsNotExist(err) { + return rest.InClusterConfig() + } + return clientcmd.BuildConfigFromFlags("", kubeconfigPath) +} + +func StartSuricataIfNotRunning(clientset *kubernetes.Clientset, nodeName string, l7Reconciler *Reconciler) { + L7mutex.Lock() + defer L7mutex.Unlock() + + // Get the node by name + node, err := clientset.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) + if err != nil { + klog.ErrorS(err, "") + } + + // Check if Suricata is running on the node + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { + checkSuricataProcess(nodeName, l7Reconciler) + } + } +} + +func checkSuricataProcess(nodeName string, l7Reconciler *Reconciler) { + currentnode := os.Getenv("NODE_NAME") + if nodeName == currentnode { + cmd := exec.Command("bash", "-c", "ps aux | grep suricata") + stdout, err := cmd.CombinedOutput() + if err != nil { + return + } + if !strings.Contains(string(stdout), "suricata -c") { + l7Reconciler.startSuricata() + } + } +} + +func createTrafficControl( + generateName string, + nsSelector, + podSelector map[string]string, + direction v1alpha2.Direction, + action v1alpha2.TrafficControlAction, + targetPort interface{}, + tcController *trafficcontrol.Controller) *v1alpha2.TrafficControl { + tc := &v1alpha2.TrafficControl{ + ObjectMeta: metav1.ObjectMeta{GenerateName: generateName}, + Spec: v1alpha2.TrafficControlSpec{ + Direction: direction, + Action: action, + }, + } + if nsSelector != nil { + tc.Spec.AppliedTo.NamespaceSelector = &metav1.LabelSelector{MatchLabels: nsSelector} + } + if podSelector != nil { + tc.Spec.AppliedTo.PodSelector = &metav1.LabelSelector{MatchLabels: podSelector} + } + tc.Spec.TargetPort.OVSInternal = targetPort.(*v1alpha2.OVSInternalPort) + trafficControlApi := tcController.CrdClient.CrdV1alpha2().TrafficControls() + tc, err := trafficControlApi.Create(context.TODO(), tc, metav1.CreateOptions{}) + if err != nil { + klog.ErrorS(err, "Failed to create TrafficControl") + } + return tc +} + +func postTrafficControl(direction v1alpha2.Direction, podLabels, namespaceLabels map[string]string, tcController *trafficcontrol.Controller) *v1alpha2.TrafficControl { + targetPort := &v1alpha2.OVSInternalPort{Name: config.L7NetworkPolicyTargetPortName} + klog.InfoS("TUSHAR VALS", "namespaceLabels", namespaceLabels, "podLabels", podLabels, "direction", direction) + var tc *v1alpha2.TrafficControl + if len(podLabels) != 0 { + tc = createTrafficControl("tc-", namespaceLabels, podLabels, direction, v1alpha2.ActionMirror, targetPort, tcController) + } else { + tc = createTrafficControl("tc-", namespaceLabels, nil, direction, v1alpha2.ActionMirror, targetPort, tcController) + } + return tc +} diff --git a/pkg/agent/controller/networkpolicy/l7engine/reconciler.go b/pkg/agent/controller/networkpolicy/l7engine/reconciler.go index b2e9b012ee5..f29929e8fb8 100644 --- a/pkg/agent/controller/networkpolicy/l7engine/reconciler.go +++ b/pkg/agent/controller/networkpolicy/l7engine/reconciler.go @@ -228,6 +228,7 @@ func (r *Reconciler) AddRule(ruleID, policyName string, vlanID uint32, l7Protoco // Add a Suricata tenant. if err := r.addBindingSuricataTenant(vlanID, rulesPath); err != nil { return fmt.Errorf("failed to add Suricata tenant for L7 rule %s of %s: %w", ruleID, policyName, err) + } return nil @@ -386,7 +387,7 @@ func (r *Reconciler) unregisterSuricataTenantHandler(tenantID, vlanID uint32) (* return r.suricataScFn(scCmd) } -func (r *Reconciler) startSuricata() { +func (r *Reconciler) startSuricata() { // Create the config file /etc/suricata/antrea.yaml for Antrea which will be included in the default Suricata config file // /etc/suricata/suricata.yaml. suricataAntreaConfigData := fmt.Sprintf(`%%YAML 1.1 @@ -405,7 +406,28 @@ outputs: types: - alert: tagged-packets: yes + - eve-log: + enabled: yes + filetype: unix_stream + filename: /var/log/antrea/networkpolicy/suricata_eve.socket + rotate-interval: day + pcap-file: false + community-id: false + community-id-seed: 0 + xff: + enabled: no + types: + - http: + extended: yes af-packet: + - interface: eth0 + threads: auto + cluster-id: 93 + cluster-type: cluster_flow + defrag: no + use-mmap: yes + tpacket-v2: yes + checksum-checks: no - interface: %[1]s threads: auto cluster-id: 80 diff --git a/pkg/agent/controller/trafficcontrol/controller.go b/pkg/agent/controller/trafficcontrol/controller.go index 1c0d8720a77..328cf8fbe2e 100644 --- a/pkg/agent/controller/trafficcontrol/controller.go +++ b/pkg/agent/controller/trafficcontrol/controller.go @@ -42,6 +42,7 @@ import ( "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/apis/crd/v1alpha2" + clientsetversioned "antrea.io/antrea/pkg/client/clientset/versioned" crdinformers "antrea.io/antrea/pkg/client/informers/externalversions/crd/v1alpha2" crdlisters "antrea.io/antrea/pkg/client/listers/crd/v1alpha2" "antrea.io/antrea/pkg/ovs/ovsconfig" @@ -112,7 +113,8 @@ type portToTCBinding struct { } type Controller struct { - ofClient openflow.Client + ofClient openflow.Client + CrdClient clientsetversioned.Interface portToTCBindings map[string]*portToTCBinding ovsBridgeClient ovsconfig.OVSBridgeClient @@ -142,6 +144,7 @@ type Controller struct { } func NewTrafficControlController(ofClient openflow.Client, + crdClient clientsetversioned.Interface, interfaceStore interfacestore.InterfaceStore, ovsBridgeClient ovsconfig.OVSBridgeClient, ovsCtlClient ovsctl.OVSCtlClient, @@ -151,6 +154,7 @@ func NewTrafficControlController(ofClient openflow.Client, podUpdateSubscriber channel.Subscriber) *Controller { c := &Controller{ ofClient: ofClient, + CrdClient: crdClient, ovsBridgeClient: ovsBridgeClient, ovsCtlClient: ovsCtlClient, interfaceStore: interfaceStore, diff --git a/pkg/agent/flowexporter/connections/conntrack_ovs.go b/pkg/agent/flowexporter/connections/conntrack_ovs.go index 52e0f89bfba..e4f4b67c8d3 100644 --- a/pkg/agent/flowexporter/connections/conntrack_ovs.go +++ b/pkg/agent/flowexporter/connections/conntrack_ovs.go @@ -32,13 +32,6 @@ import ( // Following map is for converting protocol name (string) to protocol identifier var ( - protocols = map[string]uint8{ - "icmp": 1, - "igmp": 2, - "tcp": 6, - "udp": 17, - "ipv6-icmp": 58, - } // Mapping is defined at https://github.com/torvalds/linux/blob/v5.9/include/uapi/linux/netfilter/nf_conntrack_common.h#L42 conntrackStatusMap = map[string]uint32{ "EXPECTED": uint32(1), @@ -140,7 +133,7 @@ func flowStringToAntreaConnection(flow string, zoneFilter uint16) (*flowexporter switch { case hasAnyProto(fs): // Proto identifier - proto, err := lookupProtocolMap(fs) + proto, err := flowexporter.LookupProtocolMap(fs) if err != nil { return nil, err } @@ -287,7 +280,7 @@ func flowStringToAntreaConnection(flow string, zoneFilter uint16) (*flowexporter } func hasAnyProto(text string) bool { - for proto := range protocols { + for proto := range flowexporter.Protocols { if strings.Contains(strings.ToLower(text), proto) { return true } @@ -295,17 +288,6 @@ func hasAnyProto(text string) bool { return false } -// lookupProtocolMap returns protocol identifier given protocol name -func lookupProtocolMap(name string) (uint8, error) { - name = strings.TrimSpace(name) - lowerCaseStr := strings.ToLower(name) - proto, found := protocols[lowerCaseStr] - if !found { - return 0, fmt.Errorf("unknown IP protocol specified: %s", name) - } - return proto, nil -} - func (ct *connTrackOvsCtl) GetMaxConnections() (int, error) { cmdOutput, execErr := ct.ovsctlClient.RunAppctlCmd("dpctl/ct-get-maxconns", false) if execErr != nil { diff --git a/pkg/agent/flowexporter/exporter/exporter.go b/pkg/agent/flowexporter/exporter/exporter.go index 84beeff9279..84b2b0be80f 100644 --- a/pkg/agent/flowexporter/exporter/exporter.go +++ b/pkg/agent/flowexporter/exporter/exporter.go @@ -15,10 +15,15 @@ package exporter import ( + "bufio" "context" + "encoding/json" "fmt" "hash/fnv" "net" + "os" + "strconv" + "strings" "time" ipfixentities "github.com/vmware/go-ipfix/pkg/entities" @@ -37,6 +42,7 @@ import ( "antrea.io/antrea/pkg/agent/metrics" "antrea.io/antrea/pkg/agent/openflow" "antrea.io/antrea/pkg/agent/proxy" + "antrea.io/antrea/pkg/features" "antrea.io/antrea/pkg/ipfix" "antrea.io/antrea/pkg/ovs/ovsconfig" "antrea.io/antrea/pkg/querier" @@ -54,7 +60,11 @@ import ( // can be taking a fraction of the size of connection store to approximate the // number of expired connections, while having a min and a max to handle edge cases, // e.g. min(50 + 0.1 * connectionStore.size(), 200) -const maxConnsToExport = 64 +const ( + maxConnsToExport = 64 + socketPath = "/var/log/antrea/networkpolicy/suricata_eve.socket" + // socketPath = "suricata_eve.socket" +) var ( IANAInfoElementsCommon = []string{ @@ -99,6 +109,8 @@ var ( "egressNetworkPolicyRuleAction", "tcpState", "flowType", + "isL7", + "httpVals", } AntreaInfoElementsIPv4 = append(antreaInfoElementsCommon, []string{"destinationClusterIPv4"}...) AntreaInfoElementsIPv6 = append(antreaInfoElementsCommon, []string{"destinationClusterIPv6"}...) @@ -126,6 +138,39 @@ type FlowExporter struct { conntrackPriorityQueue *priorityqueue.ExpirePriorityQueue denyPriorityQueue *priorityqueue.ExpirePriorityQueue expiredConns []flowexporter.Connection + l7events map[flowexporter.ConnectionKey]*httpEvent +} + +// Define struct to hold the L7 flow JSON values +type Http struct { + Hostname string `json:"hostname"` + URL string `json:"url"` + UserAgent string `json:"http_user_agent"` + ContentType string `json:"http_content_type"` + Method string `json:"http_method"` + Protocol string `json:"protocol"` + Status int `json:"status"` + ContentLength int `json:"length"` +} + +type JsonToEvent struct { + Timestamp string `json:"timestamp"` + FlowID int64 `json:"flow_id"` + InInterface string `json:"in_iface"` + EventType string `json:"event_type"` + VLAN []int `json:"vlan"` + SrcIP net.IP `json:"src_ip"` + SrcPort int `json:"src_port"` + DestIP net.IP `json:"dest_ip"` + DestPort int `json:"dest_port"` + Proto string `json:"proto"` + TxID int `json:"tx_id"` + HTTP Http `json:"http"` +} + +type httpEvent struct { + http [][]string + httpQueryFlag bool } func genObservationID(nodeName string) uint32 { @@ -184,6 +229,7 @@ func NewFlowExporter(ifaceStore interfacestore.InterfaceStore, proxier proxy.Pro conntrackPriorityQueue: conntrackConnStore.GetPriorityQueue(), denyPriorityQueue: denyConnStore.GetPriorityQueue(), expiredConns: make([]flowexporter.Connection, 0, maxConnsToExport*2), + l7events: make(map[flowexporter.ConnectionKey]*httpEvent), }, nil } @@ -198,6 +244,11 @@ func (exp *FlowExporter) Run(stopCh <-chan struct{}) { // Start the goroutine to poll conntrack flows. go exp.conntrackConnStore.Run(stopCh) + // Start L7 connection flow socket + if features.DefaultFeatureGate.Enabled(features.L7NetworkPolicy) { + go exp.l7Listener() + } + defaultTimeout := exp.conntrackPriorityQueue.ActiveFlowTimeout expireTimer := time.NewTimer(defaultTimeout) for { @@ -413,6 +464,7 @@ func (exp *FlowExporter) sendTemplateSet(isIPv6 bool) (int, error) { func (exp *FlowExporter) addConnToSet(conn *flowexporter.Connection) error { exp.ipfixSet.ResetSet() + connkey := flowexporter.NewConnectionKey(conn) eL := exp.elementsListv4 templateID := exp.templateIDv4 @@ -555,6 +607,10 @@ func (exp *FlowExporter) addConnToSet(conn *flowexporter.Connection) error { ie.SetStringValue(conn.TCPState) case "flowType": ie.SetUnsigned8Value(exp.findFlowType(*conn)) + case "isL7": + ie.SetStringValue(exp.getL7EventData(connkey, "httpL7")) + case "httpVals": + ie.SetStringValue(exp.getL7EventData(connkey, "httpVals")) } } err := exp.ipfixSet.AddRecord(eL, templateID) @@ -619,3 +675,138 @@ func getMinTime(t1, t2 time.Duration) time.Duration { } return t2 } + +func (exp *FlowExporter) l7Listener() { + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + klog.ErrorS(err, "Error removing Suricata socket") + } + listener, err := net.Listen("unix", socketPath) + if err != nil { + klog.ErrorS(err, "Error listening on Suricata socket") + return + } + defer listener.Close() + + klog.InfoS("L7 Listener Server started. Listening for connections...") + + for { + conn, err := listener.Accept() + if err != nil { + klog.ErrorS(err, "Error accepting Suricata connection") + continue + } + go exp.handleClientConnection(conn) + } +} + +func (exp *FlowExporter) handleClientConnection(conn net.Conn) { + defer conn.Close() + reader := bufio.NewReader(conn) + for { + buffer, err := reader.ReadBytes('\n') + if err != nil { + klog.ErrorS(err, "Error reading data", "buffer", buffer) + break + } + exp.processLog(buffer) + } +} + +func (exp *FlowExporter) processLog(data []byte) { + // Check if the event type is "http" + if eventType := extractEventType(data); eventType != "http" { + return + } + var event JsonToEvent + err := json.Unmarshal(data, &event) + if err != nil { + klog.ErrorS(err, "Error parsing JSON:", "data", data) + return + } + exp.WriteConnection(event) +} + +func extractEventType(data []byte) string { + // Define a temporary struct for extracting the event type + type Temp struct { + EventType string `json:"event_type"` + } + + // Parse the JSON string and extract the event type + var temp Temp + err := json.Unmarshal(data, &temp) + if err != nil { + klog.ErrorS(err, "Error parsing JSON for eventtype:", "data", data) + return "" + } + + return temp.EventType +} + +func (exp *FlowExporter) WriteConnection(event JsonToEvent) { + protocol, err := flowexporter.LookupProtocolMap(event.Proto) + if err != nil { + klog.ErrorS(err, "InValid Protocol type") + return + } + // Get 5-tuple information + tuple := flowexporter.Tuple{ + SourceAddress: event.SrcIP, + DestinationAddress: event.DestIP, + Protocol: protocol, + SourcePort: uint16(event.SrcPort), + DestinationPort: uint16(event.DestPort), + } + conn := flowexporter.Connection{} + conn.FlowKey = tuple + connKey := flowexporter.NewConnectionKey(&conn) + var tempArr []string + tempArr = append(tempArr, + event.HTTP.Hostname, + event.HTTP.URL, + event.HTTP.UserAgent, + event.HTTP.ContentType, + event.HTTP.Method, + event.HTTP.Protocol, + strconv.Itoa(event.HTTP.Status), + strconv.Itoa(event.HTTP.ContentLength)) + if _, found := exp.l7events[connKey]; found { + exp.l7events[connKey].http = append(exp.l7events[connKey].http, tempArr) + exp.l7events[connKey].httpQueryFlag = false + } else { + var temp httpEvent + temp.http = append(temp.http, tempArr) + temp.httpQueryFlag = false + exp.l7events[connKey] = &temp + } + klog.InfoS("TUSHAR recieved httpVals", "httpVALS", exp.l7events[connKey]) +} + +func (exp *FlowExporter) getL7EventData(connkey flowexporter.ConnectionKey, field string) string { + _, exists := exp.l7events[connkey] + if exists { + switch field { + case "httpL7": + return "true" + case "httpVals": + var temp []string + // Using '<' '>' as delimiters + klog.InfoS("TUSHAR httpVals", "httpVALS", exp.l7events[connkey].http) + for _, httpVals := range exp.l7events[connkey].http { + temp = append(temp, strings.Join(httpVals, "<>")) + } + allHttpVals := strings.Join(temp, "><") + exp.l7events[connkey].httpQueryFlag = true + exp.verifyAndDeleteEvent(connkey) + klog.InfoS("TUSHAR httpVals remaining", "httpVALS", exp.l7events) + return allHttpVals + } + } + return "" +} + +func (exp *FlowExporter) verifyAndDeleteEvent(connkey flowexporter.ConnectionKey) { + if exp.l7events[connkey].httpQueryFlag { + delete(exp.l7events, connkey) + } +} diff --git a/pkg/agent/flowexporter/exporter/exporter_test.go b/pkg/agent/flowexporter/exporter/exporter_test.go index bff4b3f4547..23fbf9038e8 100644 --- a/pkg/agent/flowexporter/exporter/exporter_test.go +++ b/pkg/agent/flowexporter/exporter/exporter_test.go @@ -16,6 +16,7 @@ package exporter import ( "context" + // "encoding/json" "net" "strings" "testing" @@ -798,3 +799,250 @@ func TestFlowExporter_findFlowType(t *testing.T) { assert.Equal(t, tc.expectedFlowType, flowType) } } + +// func TestFlowExporter_l7Listener(t *testing.T) { +// flowExp := &FlowExporter{ +// l7events: make(map[flowexporter.ConnectionKey]*Event), +// } +// go flowExp.l7Listener() +// <-time.After(100 * time.Millisecond) + +// conn, err := net.Dial("unix", socketPath) +// if err != nil { +// t.Fatalf("Failed to connect to server: %s", err) +// } +// defer conn.Close() + +// testCases := []struct { +// name string +// input Event +// eventPresent bool +// expected Event +// expectedErr error +// }{ +// { +// name: "Valid case", +// input: Event{ +// Timestamp: "2023-06-16T20:31:48.910477+0000", +// FlowID: 1, +// InInterface: "mock_interface", +// EventType: "http", +// VLAN: []int{1}, +// SrcIP: net.ParseIP("10.10.0.1"), +// SrcPort: 59920, +// DestIP: net.ParseIP("10.10.0.2"), +// DestPort: 80, +// Proto: "TCP", +// TxID: 0, +// HTTP: Http{ +// Hostname: "10.10.0.1", +// URL: "/public/", +// UserAgent: "curl/7.74.0", +// ContentType: "text/html", +// Method: "GET", +// Protocol: "HTTP/1.1", +// Status: 200, +// ContentLength: 153, +// }, +// }, +// eventPresent: true, +// expected: Event{ +// Timestamp: "2023-06-16T20:31:48.910477+0000", +// FlowID: 1, +// InInterface: "mock_interface", +// EventType: "http", +// VLAN: []int{1}, +// SrcIP: net.ParseIP("10.10.0.1"), +// SrcPort: 59920, +// DestIP: net.ParseIP("10.10.0.2"), +// DestPort: 80, +// Proto: "TCP", +// TxID: 0, +// HTTP: Http{ +// Hostname: "10.10.0.1", +// URL: "/public/", +// UserAgent: "curl/7.74.0", +// ContentType: "text/html", +// Method: "GET", +// Protocol: "HTTP/1.1", +// Status: 200, +// ContentLength: 153, +// }, +// }, +// }, { +// name: "InValid eventType", +// input: Event{ +// Timestamp: "2023-06-16T20:31:48.910477+0000", +// FlowID: 1, +// InInterface: "mock_interface", +// EventType: "mock_event", +// VLAN: []int{1}, +// SrcIP: net.ParseIP("10.10.0.1"), +// SrcPort: 59920, +// DestIP: net.ParseIP("10.10.0.2"), +// DestPort: 80, +// Proto: "TCP", +// TxID: 0, +// HTTP: Http{ +// Hostname: "10.10.0.1", +// URL: "/public/", +// UserAgent: "curl/7.74.0", +// ContentType: "text/html", +// Method: "GET", +// Protocol: "HTTP/1.1", +// Status: 200, +// ContentLength: 153, +// }, +// }, +// eventPresent: false, +// expected: Event{}, +// }, +// } +// for _, tc := range testCases { +// jsonData, err := json.Marshal(tc.input) +// if err != nil { +// t.Errorf("Error Marshaling data: %v", err) +// } +// _, err = conn.Write(jsonData) +// if err != nil { +// t.Errorf("Error writing event data: %v", err) +// } +// <-time.After(100 * time.Millisecond) +// } + +// for _, tc := range testCases { +// protocol, _ := flowexporter.LookupProtocolMap(tc.expected.Proto) +// // Get 5-tuple information +// tuple := flowexporter.Tuple{ +// SourceAddress: tc.expected.SrcIP, +// DestinationAddress: tc.expected.DestIP, +// Protocol: protocol, +// SourcePort: uint16(tc.expected.SrcPort), +// DestinationPort: uint16(tc.expected.DestPort), +// } +// // Generate deny connection and add to deny connection store +// conn := flowexporter.Connection{} +// conn.FlowKey = tuple +// connKey := flowexporter.NewConnectionKey(&conn) +// // Check if event is present in event map +// existingEvent, exists := flowExp.l7events[connKey] +// assert.Equal(t, tc.eventPresent, exists) +// if exists == true { +// assert.Equal(t, tc.expected, *existingEvent) +// } +// } +// } + +// func TestGetL7EventData(t *testing.T) { +// flowExp := &FlowExporter{ +// l7events: make(map[flowexporter.ConnectionKey]*Event), +// } + +// // 5-tuple information +// tuple := flowexporter.Tuple{ +// SourceAddress: net.IP("10.10.0.1"), +// DestinationAddress: net.IP("10.10.0.2"), +// Protocol: 6, +// SourcePort: uint16(5229), +// DestinationPort: uint16(80), +// } +// conn := flowexporter.Connection{} +// conn.FlowKey = tuple +// connKey := flowexporter.NewConnectionKey(&conn) +// testCases := []struct { +// name string +// input Event +// expected string +// }{ +// { +// name: "httpHostname", +// input: Event{ +// HTTP: Http{ +// Hostname: "10.0.0.1", +// }, +// }, +// expected: "10.0.0.1", +// }, { +// name: "httpURL", +// input: Event{ +// HTTP: Http{ +// URL: "/public/", +// }, +// }, +// expected: "/public/", +// }, { +// name: "httpUserAgent", +// input: Event{ +// HTTP: Http{ +// UserAgent: "curl/7.74.0", +// }, +// }, +// expected: "curl/7.74.0", +// }, { +// name: "httpContentType", +// input: Event{ +// HTTP: Http{ +// ContentType: "text/html", +// }, +// }, +// expected: "text/html", +// }, { +// name: "httpMethod", +// input: Event{ +// HTTP: Http{ +// Method: "GET", +// }, +// }, +// expected: "GET", +// }, +// } +// for _, tc := range testCases { +// flowExp.l7events[connKey] = &tc.input +// result := flowExp.getL7EventData(connKey, tc.name) +// assert.Equal(t, tc.expected, result) +// } + +// //Connkey not present in Event map +// conn.FlowKey.Protocol = 1 +// connKey = flowexporter.NewConnectionKey(&conn) +// result := flowExp.getL7EventData(connKey, "httpHostname") +// assert.Equal(t, "", result) +// } + +// func TestGetL7EventHttpStatus(t *testing.T) { +// flowExp := &FlowExporter{ +// l7events: make(map[flowexporter.ConnectionKey]*Event), +// } + +// // 5-tuple information +// tuple := flowexporter.Tuple{ +// SourceAddress: net.IP("10.10.0.1"), +// DestinationAddress: net.IP("10.10.0.2"), +// Protocol: 6, +// SourcePort: uint16(5229), +// DestinationPort: uint16(80), +// } +// conn := flowexporter.Connection{} +// conn.FlowKey = tuple +// connKey := flowexporter.NewConnectionKey(&conn) +// testCases := []struct { +// name string +// input Event +// expected uint16 +// }{ +// { +// name: "httpStatus", +// input: Event{ +// HTTP: Http{ +// Status: 200, +// }, +// }, +// expected: uint16(200), +// }, +// } +// for _, tc := range testCases { +// flowExp.l7events[connKey] = &tc.input +// result := flowExp.getL7EventHttpStatus(connKey, tc.name) +// assert.Equal(t, tc.expected, result) +// } +// } diff --git a/pkg/agent/flowexporter/utils.go b/pkg/agent/flowexporter/utils.go index 7b007ed1863..8bdc6ecc984 100644 --- a/pkg/agent/flowexporter/utils.go +++ b/pkg/agent/flowexporter/utils.go @@ -15,7 +15,9 @@ package flowexporter import ( + "fmt" "strconv" + "strings" "github.com/vmware/go-ipfix/pkg/registry" @@ -26,6 +28,16 @@ const ( connectionDyingFlag = uint32(1 << 9) ) +var ( + Protocols = map[string]uint8{ + "icmp": 1, + "igmp": 2, + "tcp": 6, + "udp": 17, + "ipv6-icmp": 58, + } +) + // NewConnectionKey creates 5-tuple of flow as connection key func NewConnectionKey(conn *Connection) ConnectionKey { return ConnectionKey{conn.FlowKey.SourceAddress.String(), @@ -91,3 +103,14 @@ func PolicyTypeToUint8(policyType v1beta2.NetworkPolicyType) uint8 { return registry.PolicyTypeK8sNetworkPolicy } } + +// LookupProtocolMap returns protocol identifier given protocol name +func LookupProtocolMap(name string) (uint8, error) { + name = strings.TrimSpace(name) + lowerCaseStr := strings.ToLower(name) + proto, found := Protocols[lowerCaseStr] + if !found { + return 0, fmt.Errorf("unknown IP protocol specified: %s", name) + } + return proto, nil +} \ No newline at end of file diff --git a/pkg/features/antrea_features.go b/pkg/features/antrea_features.go index 898892ff6a5..a8bac8154e8 100644 --- a/pkg/features/antrea_features.go +++ b/pkg/features/antrea_features.go @@ -125,6 +125,10 @@ const ( // Enable users to protect their applications by specifying how they are allowed to communicate with others, taking // into account application context. L7NetworkPolicy featuregate.Feature = "L7NetworkPolicy" + + // alpha: v1.13 + // Enable L7 visibility on Pods and Namespaces + L7Visibility featuregate.Feature = "L7Visibility" ) var ( @@ -159,6 +163,7 @@ var ( ExternalNode: {Default: false, PreRelease: featuregate.Alpha}, SupportBundleCollection: {Default: false, PreRelease: featuregate.Alpha}, L7NetworkPolicy: {Default: false, PreRelease: featuregate.Alpha}, + L7Visibility: {Default: false, PreRelease: featuregate.Alpha}, } // UnsupportedFeaturesOnWindows records the features not supported on @@ -182,6 +187,7 @@ var ( // in the future if it's fully tested on Windows. Multicluster: {}, L7NetworkPolicy: {}, + L7Visibility: {}, } // supportedFeaturesOnExternalNode records the features supported on an external // Node. Antrea Agent checks the enabled features if it is running on an diff --git a/pkg/flowaggregator/clickhouseclient/clickhouseclient.go b/pkg/flowaggregator/clickhouseclient/clickhouseclient.go index 1e9865b81cb..07d828fd30b 100644 --- a/pkg/flowaggregator/clickhouseclient/clickhouseclient.go +++ b/pkg/flowaggregator/clickhouseclient/clickhouseclient.go @@ -82,9 +82,12 @@ const ( throughputFromDestinationNode, reverseThroughputFromSourceNode, reverseThroughputFromDestinationNode, - clusterUUID) + clusterUUID, + isL7, + httpVals) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?)` ) // PrepareClickHouseConnection is used for unit testing @@ -343,7 +346,10 @@ func (ch *ClickHouseExportProcess) batchCommitAll(ctx context.Context) (int, err record.ThroughputFromDestinationNode, record.ReverseThroughputFromSourceNode, record.ReverseThroughputFromDestinationNode, - ch.clusterUUID) + ch.clusterUUID, + record.IsL7, + record.HttpVals, + ) if err != nil { klog.ErrorS(err, "Error when adding record") diff --git a/pkg/flowaggregator/flowlogger/logger.go b/pkg/flowaggregator/flowlogger/logger.go index e27ff151024..fba9281fbc7 100644 --- a/pkg/flowaggregator/flowlogger/logger.go +++ b/pkg/flowaggregator/flowlogger/logger.go @@ -114,6 +114,8 @@ func (fl *FlowLogger) WriteRecord(r *flowrecord.FlowRecord, prettyPrint bool) er r.EgressNetworkPolicyRuleName, egressNetworkPolicyRuleAction, egressNetworkPolicyType, + r.IsL7, + r.HttpVals, } str := strings.Join(fields, ",") diff --git a/pkg/flowaggregator/flowrecord/record.go b/pkg/flowaggregator/flowrecord/record.go index 59ddcd68785..7f765ed8274 100644 --- a/pkg/flowaggregator/flowrecord/record.go +++ b/pkg/flowaggregator/flowrecord/record.go @@ -68,6 +68,8 @@ type FlowRecord struct { ThroughputFromDestinationNode uint64 ReverseThroughputFromSourceNode uint64 ReverseThroughputFromDestinationNode uint64 + IsL7 string + HttpVals string } // GetFlowRecord converts ipfixentities.Record to FlowRecord @@ -220,6 +222,12 @@ func GetFlowRecord(record ipfixentities.Record) *FlowRecord { if revTputFromDstNode, _, ok := record.GetInfoElementWithValue("reverseThroughputFromDestinationNode"); ok { r.ReverseThroughputFromDestinationNode = revTputFromDstNode.GetUnsigned64Value() } + if isL7, _, ok := record.GetInfoElementWithValue("isL7"); ok { + r.IsL7 = isL7.GetStringValue() + } + if httpVals, _, ok := record.GetInfoElementWithValue("httpVals"); ok { + r.HttpVals = httpVals.GetStringValue() + } return r } diff --git a/pkg/flowaggregator/infoelements/elements.go b/pkg/flowaggregator/infoelements/elements.go index 8661807a792..3487d373dff 100644 --- a/pkg/flowaggregator/infoelements/elements.go +++ b/pkg/flowaggregator/infoelements/elements.go @@ -57,6 +57,8 @@ var ( "egressNetworkPolicyRuleAction", "tcpState", "flowType", + "isL7", + "httpVals", } AntreaInfoElementsIPv4 = append(AntreaInfoElementsCommon, []string{"destinationClusterIPv4"}...) AntreaInfoElementsIPv6 = append(AntreaInfoElementsCommon, []string{"destinationClusterIPv6"}...) diff --git a/pkg/flowaggregator/s3uploader/s3uploader.go b/pkg/flowaggregator/s3uploader/s3uploader.go index 03710e5085b..5bde92d0dd8 100644 --- a/pkg/flowaggregator/s3uploader/s3uploader.go +++ b/pkg/flowaggregator/s3uploader/s3uploader.go @@ -480,4 +480,8 @@ func writeRecord(w io.Writer, r *flowrecord.FlowRecord, clusterUUID string) { io.WriteString(w, clusterUUID) io.WriteString(w, ",") io.WriteString(w, fmt.Sprintf("%d", time.Now().Unix())) + io.WriteString(w, ",") + io.WriteString(w, r.IsL7) + io.WriteString(w, ",") + io.WriteString(w, r.HttpVals) }