From 4c362fd543c00aeb4257ac6bcc64fe102e05f31e Mon Sep 17 00:00:00 2001
From: Tobias Klauser <tklauser@distanz.ch>
Date: Fri, 21 Feb 2020 15:22:29 +0100
Subject: [PATCH 09/10] option: reduce default number for TCP CT and NAT table
 max entries

Commit e824a86bba21 ("daemon: Allow configuration of CT max entries")
bumped the default value to 1000000 in order to ease upgrades from
Cilium 1.2. In the helm charts, the value was again set to 512KB via the
`ct-global-max-entries-tcp` option. However, if Cilium is not deployed
via helm charts (e.g. when running as a systemd service in the devel
VM) the large default number of entries is used.

Set the default value for `bpf-ct-global-tcp-max` to 512KB again and
instead advise users in the helm chart comments to set it to 1000000 in
case they're upgrading or changed the size manually using Helm.

Since the default value of `bpf-nat-global-max` for the NAT table size
is derived from the default for `bpf-ct-global-tcp-max`, this commit
will also decrease the the NAT table size to 512K.

Document possible consequences of upgrading Cilium installations with
larger TCP CT and NAT table sizes.

This saves about ~150MB of memory at runtime.

Updates #10056

Signed-off-by: Tobias Klauser <tklauser@distanz.ch>
---
 Documentation/cmdref/cilium-agent.md          |   4 +-
 Documentation/install/upgrade.rst             | 315 ++++++++++++++++++
 .../charts/config/templates/configmap.yaml    |   4 +-
 install/kubernetes/cilium/values.yaml         |   2 +-
 install/kubernetes/quick-install.yaml         |   2 +-
 pkg/option/config.go                          |   2 +-
 6 files changed, 322 insertions(+), 7 deletions(-)

diff --git a/Documentation/cmdref/cilium-agent.md b/Documentation/cmdref/cilium-agent.md
index 8925944db..e7e7a755a 100644
--- a/Documentation/cmdref/cilium-agent.md
+++ b/Documentation/cmdref/cilium-agent.md
@@ -25,14 +25,14 @@ cilium-agent [flags]
       --blacklist-conflicting-routes                  Don't blacklist IP allocations conflicting with local non-cilium routes (default true)
       --bpf-compile-debug                             Enable debugging of the BPF compilation process
       --bpf-ct-global-any-max int                     Maximum number of entries in non-TCP CT table (default 262144)
-      --bpf-ct-global-tcp-max int                     Maximum number of entries in TCP CT table (default 1000000)
+      --bpf-ct-global-tcp-max int                     Maximum number of entries in TCP CT table (default 524288)
       --bpf-ct-timeout-regular-any duration           Timeout for entries in non-TCP CT table (default 1m0s)
       --bpf-ct-timeout-regular-tcp duration           Timeout for established entries in TCP CT table (default 6h0m0s)
       --bpf-ct-timeout-regular-tcp-fin duration       Teardown timeout for entries in TCP CT table (default 10s)
       --bpf-ct-timeout-regular-tcp-syn duration       Establishment timeout for entries in TCP CT table (default 1m0s)
       --bpf-ct-timeout-service-any duration           Timeout for service entries in non-TCP CT table (default 1m0s)
       --bpf-ct-timeout-service-tcp duration           Timeout for established service entries in TCP CT table (default 6h0m0s)
-      --bpf-nat-global-max int                        Maximum number of entries for the global BPF NAT table (default 841429)
+      --bpf-nat-global-max int                        Maximum number of entries for the global BPF NAT table (default 524288)
       --bpf-policy-map-max int                        Maximum number of entries in endpoint policy map (per endpoint) (default 16384)
       --bpf-root string                               Path to BPF filesystem
       --certificates-directory string                 Root directory to find certificates specified in L7 TLS policy enforcement (default "/var/run/cilium/certs")
diff --git a/Documentation/install/upgrade.rst b/Documentation/install/upgrade.rst
index 75c162e9d..001b2aa86 100644
--- a/Documentation/install/upgrade.rst
+++ b/Documentation/install/upgrade.rst
@@ -311,6 +311,321 @@ Annotations:
    upgrade. Connections should successfully re-establish without requiring
    clients to reconnect.
 
+.. _1.8_upgrade_notes:
+
+1.8 Upgrade Notes
+-----------------
+
+.. _current_release_required_changes:
+
+.. _1.8_required_changes:
+
+IMPORTANT: Changes required before upgrading to 1.8.0
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. warning::
+
+   Do not upgrade to 1.8.0 before reading the following section and completing
+   the required steps.
+
+* While operating in direct-routing mode (``--tunnel=disabled``), traffic with
+  a destination address matching a particular CIDR is automatically excluded
+  from being masqueraded. So far, this CIDR consisted of
+  ``<alloc-cidr>/<size>`` where the size could be set with the option
+  ``--ipv4-cluster-cidr-mask-size``. This was not always desirable and
+  limiting, therefore Cilium 1.6 had already introduced the option
+  ``--native-routing-cidr`` allowing to explicitly specify the CIDR for native
+  routing. With Cilium 1.8, the option ``--ipv4-cluster-cidr-mask-size`` is
+  being deprecated and all users must use the option ``--native-routing-cidr``
+  instead.
+
+  .. note:: The ENI IPAM mode automatically derives the native routing CIDR so
+            no action is required.
+
+Deprecated options
+~~~~~~~~~~~~~~~~~~
+
+* ``keep-bpf-templates``: This option no longer has any effect due to the BPF
+  assets not being compiled into the cilium-agent binary anymore. The option is
+  deprecated and will be removed in Cilium 1.9.
+* ``access-log``: L7 access logs have been available via Hubble since Cilium
+  1.6. The ``access-log`` option to log to a file has been removed.
+* ``--disable-k8s-services`` option from cilium-agent has been deprecated
+  and will be removed in Cilium 1.9. 
+
+Renamed Metrics
+~~~~~~~~~~~~~~~
+
+The following metrics have been renamed:
+
+* ``cilium_operator_eni_ips`` to ``cilium_operator_ipam_ips``
+* ``cilium_operator_eni_allocation_ops`` to ``cilium_operator_ipam_allocation_ops``
+* ``cilium_operator_eni_interface_creation_ops`` to ``cilium_operator_ipam_interface_creation_ops``
+* ``cilium_operator_eni_available`` to ``cilium_operator_ipam_available``
+* ``cilium_operator_eni_nodes_at_capacity`` to ``cilium_operator_ipam_nodes_at_capacity``
+* ``cilium_operator_eni_resync_total`` to ``cilium_operator_ipam_resync_total``
+* ``cilium_operator_eni_aws_api_duration_seconds`` to ``cilium_operator_ipam_api_duration_seconds``
+* ``cilium_operator_eni_ec2_rate_limit_duration_seconds`` to ``cilium_operator_ipam_api_rate_limit_duration_seconds``
+
+Deprecated cilium-operator options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* ``metrics-address``: This option is being deprecated and a new flag is
+  introduced to replace its usage. The new option is ``operator-prometheus-serve-addr``.
+  This old option will be removed in Cilium 1.9
+
+* ``ccnp-node-status-gc``: This option is being deprecated. Disabling CCNP node
+  status GC can be done with ``cnp-node-status-gc-interval=0``. (Note that this
+  is not a typo, it is meant to be ``cnp-node-status-gc-interval``).
+  This old option will be removed in Cilium 1.9
+
+* ``cnp-node-status-gc``: This option is being deprecated. Disabling CNP node
+  status GC can be done with ``cnp-node-status-gc-interval=0``.
+  This old option will be removed in Cilium 1.9
+
+* ``cilium-endpoint-gc``: This option is being deprecated. Disabling cilium
+  endpoint GC can be done with ``cilium-endpoint-gc-interval=0``.
+  This old option will be removed in Cilium 1.9
+
+* ``api-server-port``: This option is being deprecated. The API Server address
+  and port can be enabled with ``operator-api-serve-addr=127.0.0.1:9234``
+  or ``operator-api-serve-addr=[::1]:9234`` for IPv6-only clusters.
+  This old option will be removed in Cilium 1.9
+
+* ``eni-parallel-workers``: This option in the Operator has been renamed to
+  ``parallel-alloc-workers``. The obsolete option name ``eni-parallel-workers``
+  has been deprecated and will be removed in v1.9.
+
+* ``aws-client-burst``: This option in the Operator has been renamed to
+  ``limit-ipam-api-burst``. The obsolete option name ``aws-client-burst`` has been
+  deprecated and will be removed in v1.9.
+
+* ``aws-client-qps``: This option in the Operator has been renamed to
+  ``limit-ipam-api-qps``. The obsolete option name ``aws-client-qps`` has been
+  deprecated and will be removed in v1.9.
+
+Removed options
+~~~~~~~~~~~~~~~
+
+* ``enable-legacy-services``: This option was deprecated in Cilium 1.6 and is
+  now removed.
+
+Removed helm options
+~~~~~~~~~~~~~~~~~~~~
+* ``operator.synchronizeK8sNodes``: was removed and replaced with ``global.synchronizeK8sNodes``
+
+Removed resource fields
+~~~~~~~~~~~~~~~~~~~~~~~
+
+* The fields ``CiliumEndpoint.Status.Status``,
+  ``CiliumEndpoint.Status.Spec``, and ``EndpointIdentity.LabelsSHA256``,
+  deprecated in 1.4, have been removed.
+
+=======
+.. _1.8_upgrade_notes:
+
+1.8 Upgrade Notes
+-----------------
+
+.. _current_release_required_changes:
+
+.. _1.8_required_changes:
+
+IMPORTANT: Changes required before upgrading to 1.8.0
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. warning::
+
+   Do not upgrade to 1.8.0 before reading the following section and completing
+   the required steps.
+
+* While operating in direct-routing mode (``--tunnel=disabled``), traffic with
+  a destination address matching a particular CIDR is automatically excluded
+  from being masqueraded. So far, this CIDR consisted of
+  ``<alloc-cidr>/<size>`` where the size could be set with the option
+  ``--ipv4-cluster-cidr-mask-size``. This was not always desirable and
+  limiting, therefore Cilium 1.6 had already introduced the option
+  ``--native-routing-cidr`` allowing to explicitly specify the CIDR for native
+  routing. With Cilium 1.8, the option ``--ipv4-cluster-cidr-mask-size`` is
+  being deprecated and all users must use the option ``--native-routing-cidr``
+  instead.
+
+  .. note:: The ENI IPAM mode automatically derives the native routing CIDR so
+            no action is required.
+
+Upgrading from >=1.7.0 to 1.8.y
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Since Cilium 1.5, the TCP connection tracking table size parameter
+  ``bpf-ct-global-tcp-max`` in the daemon was set to the default value
+  ``1000000`` to retain backwards compatibility with previous versions. In
+  Cilium 1.8 the default value is set to 512K by default in order to reduce the
+  agent memory consumption.
+
+  If Cilium was deployed using Helm, the new default value of 512K was already
+  effective in Cilium 1.6 or later, unless it was manually configured to a
+  different value.
+
+  If the table size was configured to a value different from 512K in the
+  previous installation, ongoing connections will be disrupted during the
+  upgrade. To avoid connection breakage, ``bpf-ct-global-tcp-max`` needs to be
+  manually adjusted.
+
+  To check whether any action is required the following command can be used to
+  check the currently configured maximum number of TCP conntrack entries:
+
+  .. code:: bash
+
+     sudo grep -R CT_MAP_SIZE_TCP /var/run/cilium/state/templates/
+
+  If the maximum number is 524288, no action is required. If the number is
+  different, ``bpf-ct-global-tcp-max`` needs to be adjusted in the `ConfigMap`
+  to the value shown by the command above (100000 in the example below):
+
+.. tabs::
+  .. group-tab:: kubectl
+
+    .. parsed-literal::
+
+      helm template cilium \\
+      --namespace=kube-system \\
+      ...
+      --set global.bpf.ctTcpMax=100000
+      ...
+      > cilium.yaml
+      kubectl apply -f cilium.yaml
+
+  .. group-tab:: Helm
+
+    .. parsed-literal::
+
+      helm upgrade cilium --namespace=kube-system \\
+      --set global.bpf.ctTcpMax=100000
+
+* The default value for the NAT table size parameter ``bpf-nat-global-max`` in
+  the daemon is derived from the default value of the conntrack table size
+  parameter ``bpf-ct-global-tcp-max``. Since the latter was changed (see
+  above), the default NAT table size decreased from ~820K to 512K.
+
+  The NAT table is only used if either BPF NodePort (``enable-node-port``
+  parameter) or masquerading (``masquerade`` parameter) are enabled. No action
+  is required if neither of the parameters is enabled.
+
+  If either of the parameters is enabled, ongoing connections will be disrupted
+  during the upgrade. In order to avoid connection breakage,
+  ``bpf-nat-global-max`` needs to be manually adjusted.
+
+  To check whether any adjustment is required the following command can be used
+  to check the currently configured maximum number of NAT table entries:
+
+  .. code:: bash
+
+     sudo grep -R SNAT_MAPPING_IPV[46]_SIZE /var/run/cilium/state/globals/
+
+  If the command does not return any value or if the returned maximum number is
+  524288, no action is required. If the number is different,
+  ``bpf-nat-global-max`` needs to be adjusted in the `ConfigMap` to the value
+  shown by the command above (841429 in the example below):
+
+.. tabs::
+  .. group-tab:: kubectl
+
+    .. parsed-literal::
+
+      helm template cilium \\
+      --namespace=kube-system \\
+      ...
+      --set global.bpf.natMax=841429
+      ...
+      > cilium.yaml
+      kubectl apply -f cilium.yaml
+
+  .. group-tab:: Helm
+
+    .. parsed-literal::
+
+      helm upgrade cilium --namespace=kube-system \\
+      --set global.bpf.natMax=841429
+
+Deprecated options
+~~~~~~~~~~~~~~~~~~
+
+* ``keep-bpf-templates``: This option no longer has any effect due to the BPF
+  assets not being compiled into the cilium-agent binary anymore. The option is
+  deprecated and will be removed in Cilium 1.9.
+* ``access-log``: L7 access logs have been available via Hubble since Cilium
+  1.6. The ``access-log`` option to log to a file has been removed.
+* ``--disable-k8s-services`` option from cilium-agent has been deprecated
+  and will be removed in Cilium 1.9.
+
+Renamed Metrics
+~~~~~~~~~~~~~~~
+
+The following metrics have been renamed:
+
+* ``cilium_operator_eni_ips`` to ``cilium_operator_ipam_ips``
+* ``cilium_operator_eni_allocation_ops`` to ``cilium_operator_ipam_allocation_ops``
+* ``cilium_operator_eni_interface_creation_ops`` to ``cilium_operator_ipam_interface_creation_ops``
+* ``cilium_operator_eni_available`` to ``cilium_operator_ipam_available``
+* ``cilium_operator_eni_nodes_at_capacity`` to ``cilium_operator_ipam_nodes_at_capacity``
+* ``cilium_operator_eni_resync_total`` to ``cilium_operator_ipam_resync_total``
+* ``cilium_operator_eni_aws_api_duration_seconds`` to ``cilium_operator_ipam_api_duration_seconds``
+* ``cilium_operator_eni_ec2_rate_limit_duration_seconds`` to ``cilium_operator_ipam_api_rate_limit_duration_seconds``
+
+Deprecated cilium-operator options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* ``metrics-address``: This option is being deprecated and a new flag is
+  introduced to replace its usage. The new option is ``operator-prometheus-serve-addr``.
+  This old option will be removed in Cilium 1.9
+
+* ``ccnp-node-status-gc``: This option is being deprecated. Disabling CCNP node
+  status GC can be done with ``cnp-node-status-gc-interval=0``. (Note that this
+  is not a typo, it is meant to be ``cnp-node-status-gc-interval``).
+  This old option will be removed in Cilium 1.9
+
+* ``cnp-node-status-gc``: This option is being deprecated. Disabling CNP node
+  status GC can be done with ``cnp-node-status-gc-interval=0``.
+  This old option will be removed in Cilium 1.9
+
+* ``cilium-endpoint-gc``: This option is being deprecated. Disabling cilium
+  endpoint GC can be done with ``cilium-endpoint-gc-interval=0``.
+  This old option will be removed in Cilium 1.9
+
+* ``api-server-port``: This option is being deprecated. The API Server address
+  and port can be enabled with ``operator-api-serve-addr=127.0.0.1:9234``
+  or ``operator-api-serve-addr=[::1]:9234`` for IPv6-only clusters.
+  This old option will be removed in Cilium 1.9
+
+* ``eni-parallel-workers``: This option in the Operator has been renamed to
+  ``parallel-alloc-workers``. The obsolete option name ``eni-parallel-workers``
+  has been deprecated and will be removed in v1.9.
+
+* ``aws-client-burst``: This option in the Operator has been renamed to
+  ``limit-ipam-api-burst``. The obsolete option name ``aws-client-burst`` has been
+  deprecated and will be removed in v1.9.
+
+* ``aws-client-qps``: This option in the Operator has been renamed to
+  ``limit-ipam-api-qps``. The obsolete option name ``aws-client-qps`` has been
+  deprecated and will be removed in v1.9.
+
+Removed options
+~~~~~~~~~~~~~~~
+
+* ``enable-legacy-services``: This option was deprecated in Cilium 1.6 and is
+  now removed.
+
+Removed helm options
+~~~~~~~~~~~~~~~~~~~~
+* ``operator.synchronizeK8sNodes``: was removed and replaced with ``global.synchronizeK8sNodes``
+
+Removed resource fields
+~~~~~~~~~~~~~~~~~~~~~~~
+
+* The fields ``CiliumEndpoint.Status.Status``,
+  ``CiliumEndpoint.Status.Spec``, and ``EndpointIdentity.LabelsSHA256``,
+  deprecated in 1.4, have been removed.
+
 .. _1.7_upgrade_notes:
 
 1.7 Upgrade Notes
diff --git a/install/kubernetes/cilium/charts/config/templates/configmap.yaml b/install/kubernetes/cilium/charts/config/templates/configmap.yaml
index 0180b17d8..77fa80523 100644
--- a/install/kubernetes/cilium/charts/config/templates/configmap.yaml
+++ b/install/kubernetes/cilium/charts/config/templates/configmap.yaml
@@ -118,7 +118,7 @@ data:
   # Only effective when monitor aggregation is set to "medium" or higher.
   monitor-aggregation-flags: {{ .Values.global.bpf.monitorFlags }}
 
-  # ct-global-max-entries-* specifies the maximum number of connections
+  # bpf-ct-global-*-max specifies the maximum number of connections
   # supported across all endpoints, split by protocol: tcp or other. One pair
   # of maps uses these values for IPv4 connections, and another pair of maps
   # use these values for IPv6 connections.
@@ -128,7 +128,7 @@ data:
   # policy drops or a change in loadbalancing decisions for a connection.
   #
   # For users upgrading from Cilium 1.2 or earlier, to minimize disruption
-  # during the upgrade process, comment out these options.
+  # during the upgrade process, set bpf-ct-global-tcp-max to 1000000.
   bpf-ct-global-tcp-max: "{{ .Values.global.bpf.ctTcpMax }}"
   bpf-ct-global-any-max: "{{ .Values.global.bpf.ctAnyMax }}"
 
diff --git a/install/kubernetes/cilium/values.yaml b/install/kubernetes/cilium/values.yaml
index 9409f90e7..302e7b4c3 100644
--- a/install/kubernetes/cilium/values.yaml
+++ b/install/kubernetes/cilium/values.yaml
@@ -210,7 +210,7 @@ global:
     ctAnyMax: 262144
 
     # natMax is the maximum number of entries for the NAT table
-    natMax: 841429
+    natMax: 524288
 
     # policyMapMax is the maximum number of entries in endpoint policy map (per endpoint)
     policyMapMax: 16384
diff --git a/install/kubernetes/quick-install.yaml b/install/kubernetes/quick-install.yaml
index cca20a800..ab2f4a261 100644
--- a/install/kubernetes/quick-install.yaml
+++ b/install/kubernetes/quick-install.yaml
@@ -71,7 +71,7 @@ data:
   # policy drops or a change in loadbalancing decisions for a connection.
   #
   # For users upgrading from Cilium 1.2 or earlier, to minimize disruption
-  # during the upgrade process, comment out these options.
+  # during the upgrade process, set bpf-ct-global-tcp-max to 1000000.
   bpf-ct-global-tcp-max: "524288"
   bpf-ct-global-any-max: "262144"
 
diff --git a/pkg/option/config.go b/pkg/option/config.go
index df24ad98b..187a38286 100644
--- a/pkg/option/config.go
+++ b/pkg/option/config.go
@@ -440,7 +440,7 @@ const (
 
 	// CTMapEntriesGlobalTCP retains the Cilium 1.2 (or earlier) size to
 	// minimize disruption during upgrade.
-	CTMapEntriesGlobalTCPDefault = 1000000
+	CTMapEntriesGlobalTCPDefault = 2 << 18 // 512Ki
 	CTMapEntriesGlobalAnyDefault = 2 << 17 // 256Ki
 	CTMapEntriesGlobalTCPName    = "bpf-ct-global-tcp-max"
 	CTMapEntriesGlobalAnyName    = "bpf-ct-global-any-max"
-- 
2.26.2

