I created a cluster with ceph 16.2.7 and added OSD with cluster expansion then I removed some of OSDs from the crush map according to the following steps. Also, the option --all-available-devices is unmanaged.
enter image description here
#ceph orch apply osd --all-available-devices --unmanaged=true
#ceph osd out osd.x
#ceph osd down osd.x
#systemctl stop ceph-d813d6b4-6d3c-11ec-a97e-000c2992a0d6#osd.x.service
#ceph osd crush rm osd.x
#ceph osd rm osd.x
#ceph auth rm osd.x
#cd /var/lib/ceph/d813d6b4-6d3c-11ec-a97e-000c2992a0d6/
#rm osd.x -rf
# cd /etc/systemd/system/ceph-d813d6b4-6d3c-11ec-a97e-000c2992a0d6.target.wants/
# rm ceph-d813d6b4-6d3c-11ec-a97e-000c2992a0d6#osd.x.service
#lvremove /dev/ceph-*
But when I removed the LVMs related to the deleted OSDs. OSDs that were removed were restored automatically. I do not want this to happen. I want to manually create OSD on disk. Can anyone explain this to me?
[root#ceph2-node-01 ~]# ceph orch ls --export --format yaml
service_type: alertmanager
service_name: alertmanager
placement:
count: 3
label: mon
---
service_type: crash
service_name: crash
placement:
host_pattern: '*'
--- !!python/object:ceph.deployment.service_spec.MonitoringSpec
config: null
networks: []
placement: !!python/object:ceph.deployment.service_spec.PlacementSpec
count: 3
count_per_host: null
host_pattern: null
hosts:
- !!python/object/new:ceph.deployment.service_spec.HostPlacementSpec
- ceph2-node-02
- ''
- ''
- !!python/object/new:ceph.deployment.service_spec.HostPlacementSpec
- ceph2-node-03
- ''
- ''
label: null
port: null
preview_only: false
service_id: null
service_type: grafana
unmanaged: false
---
service_type: mgr
service_name: mgr
placement:
count: 2
---
service_type: mon
service_name: mon
placement:
count: 5
--- !!python/object:ceph.deployment.service_spec.MonitoringSpec
config: null
networks: []
placement: !!python/object:ceph.deployment.service_spec.PlacementSpec
count: null
count_per_host: null
host_pattern: '*'
hosts: []
label: null
port: null
preview_only: false
service_id: null
service_type: node-exporter
unmanaged: false
---
service_type: osd
service_id: all-available-devices
service_name: osd.all-available-devices
placement:
host_pattern: '*'
unmanaged: true
spec:
data_devices:
all: true
filter_logic: AND
objectstore: bluestore
---
service_type: osd
service_id: dashboard-admin-1642344788791
service_name: osd.dashboard-admin-1642344788791
placement:
host_pattern: '*'
spec:
data_devices:
rotational: true
db_devices:
rotational: false
db_slots: 2
filter_logic: AND
objectstore: bluestore
--- !!python/object:ceph.deployment.service_spec.MonitoringSpec
config: null
networks: []
placement: !!python/object:ceph.deployment.service_spec.PlacementSpec
count: 3
count_per_host: null
host_pattern: null
hosts: []
label: mon
port: null
preview_only: false
service_id: null
service_type: prometheus
unmanaged: false
service_type: osd
service_id: example_drvgrp_name
placement:
hosts:
ses-node2
ses-node3
encrypted: true
unmanaged: true
Related
Im trying to deploy telegraf helm chart on kubernetes.
helm upgrade --install telegraf-instance -f values.yaml influxdata/telegraf
When I add modbus input plugin with holding_register i get error
[telegraf] Error running agent: Error loading config file /etc/telegraf/telegraf.conf: Error parsing data: line 49: key `name’ is in conflict with line 2fd
my values.yaml like below
## Default values.yaml for Telegraf
## This is a YAML-formatted file.
## ref: https://hub.docker.com/r/library/telegraf/tags/
replicaCount: 1
image:
repo: "telegraf"
tag: "1.21.4"
pullPolicy: IfNotPresent
podAnnotations: {}
podLabels: {}
imagePullSecrets: []
args: []
env:
- name: HOSTNAME
value: "telegraf-polling-service"
resources: {}
nodeSelector: {}
affinity: {}
tolerations: []
service:
enabled: true
type: ClusterIP
annotations: {}
rbac:
create: true
clusterWide: false
rules: []
serviceAccount:
create: false
name:
annotations: {}
config:
agent:
interval: 60s
round_interval: true
metric_batch_size: 1000000
metric_buffer_limit: 100000000
collection_jitter: 0s
flush_interval: 60s
flush_jitter: 0s
precision: ''
hostname: '9825128'
omit_hostname: false
processors:
- enum:
mapping:
field: "status"
dest: "status_code"
value_mappings:
healthy: 1
problem: 2
critical: 3
inputs:
- modbus:
name: "PS MAIN ENGINE"
controller: 'tcp://192.168.0.101:502'
slave_id: 1
holding_registers:
- name: "Coolant Level"
byte_order: CDAB
data_type: FLOAT32
scale: 0.001
address: [51410, 51411]
- modbus:
name: "SB MAIN ENGINE"
controller: 'tcp://192.168.0.102:502'
slave_id: 1
holding_registers:
- name: "Coolant Level"
byte_order: CDAB
data_type: FLOAT32
scale: 0.001
address: [51410, 51411]
outputs:
- influxdb_v2:
token: token
organization: organisation
bucket: bucket
urls:
- "url"
metrics:
health:
enabled: true
service_address: "http://:8888"
threshold: 5000.0
internal:
enabled: true
collect_memstats: false
pdb:
create: true
minAvailable: 1
Problem resolved by doing the following steps
deleted config section of my values.yaml
added my telegraf.conf to /additional_config path
added configmap to kubernetes with the following command
kubectl create configmap external-config --from-file=/additional_config
added the following command to values.yaml
volumes:
- name: my-config
configMap:
name: external-config
volumeMounts:
- name: my-config
mountPath: /additional_config
args:
- "--config=/etc/telegraf/telegraf.conf"
- "--config-directory=/additional_config"
I am working on a use-case where I need to add a new container in jupyterhub pod , This new container (sidecontainer) monitors jupyterhub directories.
Jupyterhub container while comming up creates a dynamic PV , see the section below
singleuser:
podNameTemplate:
extraTolerations: []
nodeSelector: {}
extraNodeAffinity:
required: []
preferred: []
extraPodAffinity:
required: []
preferred: []
extraPodAntiAffinity:
required: []
preferred: []
networkTools:
image:
name: jupyterhub/k8s-network-tools
tag: "set-by-chartpress"
pullPolicy:
pullSecrets: []
resources: {}
cloudMetadata:
# block set to true will append a privileged initContainer using the
# iptables to block the sensitive metadata server at the provided ip.
blockWithIptables: true
ip: 169.254.169.254
networkPolicy:
enabled: true
ingress: []
egress:
# Required egress to communicate with the hub and DNS servers will be
# augmented to these egress rules.
#
# This default rule explicitly allows all outbound traffic from singleuser
# pods, except to a typical IP used to return metadata that can be used by
# someone with malicious intent.
- to:
- ipBlock:
cidr: 0.0.0.0/0
except:
- 169.254.169.254/32
interNamespaceAccessLabels: ignore
allowedIngressPorts: []
events: true
extraAnnotations: {}
extraLabels:
hub.jupyter.org/network-access-hub: "true"
extraFiles: {}
extraEnv: {}
lifecycleHooks: {}
initContainers: []
extraContainers: []
uid: 1000
fsGid: 100
serviceAccountName:
storage:
type: dynamic
extraLabels: {}
extraVolumes: []
extraVolumeMounts: []
static:
pvcName:
subPath: "{username}"
capacity: 10Gi
homeMountPath: /home/jovyan
dynamic:
storageClass:
pvcNameTemplate: claim-{username}{servername}
volumeNameTemplate: volume-{username}{servername}
storageAccessModes: [ReadWriteOnce]
image:
name: jupyterhub/k8s-singleuser-sample
tag: "set-by-chartpress"
pullPolicy:
pullSecrets: []
startTimeout: 300
cpu:
limit:
guarantee:
memory:
limit:
guarantee: 1G
extraResource:
limits: {}
guarantees: {}
cmd:
defaultUrl:
extraPodConfig: {}
profileList: []
I have added my new container in the extraContainer section of deployment file , my container does start but the dynamic PV is not mounted on that container.
Is the use-case I am trying to achieve technically possible at kubernetes level.
full yaml file is present here for reference
https://github.com/jupyterhub/zero-to-jupyterhub-k8s/blob/main/jupyterhub/values.yaml
Config Map for reference
singleuser:
baseUrl: /
cloudMetadata:
enabled: true
ip: xx.xx.xx.xx
cpu: {}
events: true
extraAnnotations: {}
extraConfigFiles:
config_files:
- cm_key: ''
content: ''
file_path: ''
- cm_key: ''
content: ''
file_path: ''
enabled: false
extraContainers:
- image: 'docker:19.03-rc-dind'
lifecycle:
postStart:
exec:
command:
- sh
- '-c'
- update-ca-certificates; echo Certificates Updated
name: dind
securityContext:
privileged: true
volumeMounts:
- mountPath: /var/lib/docker
name: dind-storage
- mountPath: /usr/local/share/ca-certificates/
name: docker-cert
extraEnv:
ACTUAL_HADOOP_CONF_DIR: ''
ACTUAL_HIVE_CONF_DIR: ''
ACTUAL_SPARK_CONF_DIR: ''
CDH_PARCEL_DIR: ''
DOCKER_HOST: ''
JAVA_HOME:
LIVY_URL: ''
SPARK2_PARCEL_DIR: ''
extraLabels:
hub.jupyter.org/network-access-hub: 'true'
extraNodeAffinity:
preferred: []
required: []
extraPodAffinity:
preferred: []
required: []
extraPodAntiAffinity:
preferred: []
required: []
extraPodConfig: {}
extraResource:
guarantees: {}
limits: {}
extraTolerations: []
fsGid: 0
image:
name: >-
/jupyterhub/jpt-spark-magic
pullPolicy: IfNotPresent
tag: xxx
imagePullSecret:
email: null
enabled: false
registry: null
username: null
initContainers: []
lifecycleHooks: {}
memory:
guarantee: 1G
networkPolicy:
egress:
- to:
- ipBlock:
cidr: 0.0.0.0/0
except:
- 169.254.169.254/32
enabled: false
ingress: []
networkTools:
image:
name: >-
/k8s-hub-multispawn
pullPolicy: IfNotPresent
tag: '12345'
nodeSelector: {}
profileList:
- description: Python for data enthusiasts
display_name: 0
kubespawner_override:
cmd:
- jpt-entry-cmd.sh
cpu_limit: 1
environment:
XYZ_SERVICE_URL: 'http://XYZ:8080'
CURL_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
DOCKER_HOST: 'tcp://localhost:2375'
HADOOP_CONF_DIR: /etc/hadoop/conf
HADOOP_HOME: /usr/hdp/3.1.5.6091-7/hadoop/
HDP_DIR: /usr/hdp/3.1.5.6091-7
HDP_HOME_DIR: /usr/hdp/3.1.5.6091-7
HDP_VERSION: 3.1.5.6091-7
HIVE_CONF_DIR: /usr/hdp/3.1.5.6091-7/hive
HIVE_HOME: /usr/hdp/3.1.5.6091-7/hive
INTEGRATION_ENV: HDP3
JAVA_HOME: /usr/jdk64/jdk1.8.0_112
LD_LIBRARY_PATH: >-
/usr/hdp/3.1.5.6091-7/hadoop/lib/native:/usr/jdk64/jdk1.8.0_112/jre:/usr/hdp/3.1.5.6091-7/usr/lib/:/usr/hdp/3.1.5.6091-7/usr/lib/
LIVY_URL: 'http://ammaster01.fake.org:8999'
MLFLOW_TRACKING_URI: 'http://mlflow:5100'
NO_PROXY: mlflow
SPARK_CONF_DIR: /etc/spark2/conf
SPARK_HOME: /usr/hdp/3.1.5.6091-7/spark2
SPARK2_PARCEL_DIR: /usr/hdp/3.1.5.6091-7/spark2
TOOLS_BASE_PATH: /usr/local/bin
image: >-
/jupyterhub/jpt-spark-magic:1.1.2
mem_limit: 4096M
uid: 0
- description: R for data enthusiasts
display_name: 1
kubespawner_override:
cmd:
- start-all.sh
environment:
XYZ_SERVICE_URL: 'http://XYZ-service:8080'
DISABLE_AUTH: 'true'
XYZ: /home/rstudio/kitematic
image: '/jupyterhub/rstudio:364094'
uid: 0
- description: Python for data enthusiasts test2
display_name: 2
kubespawner_override:
cmd:
- jpt-entry-cmd.sh
cpu_limit: 4
environment:
XYZ_SERVICE_URL: 'http://XYZ-service:8080'
CURL_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
DOCKER_HOST: 'tcp://localhost:2375'
HADOOP_CONF_DIR: /etc/hadoop/conf
HADOOP_HOME: /usr/hdp/3.1.5.6091-7/hadoop/
HDP_DIR: /usr/hdp/3.1.5.6091-7
HDP_HOME_DIR: /usr/hdp/3.1.5.6091-7
HDP_VERSION: 3.1.5.6091-7
HIVE_CONF_DIR: /usr/hdp/3.1.5.6091-7/hive
HIVE_HOME: /usr/hdp/3.1.5.6091-7/hive
INTEGRATION_ENV: HDP3
JAVA_HOME: /usr/jdk64/jdk1.8.0_112
LD_LIBRARY_PATH: >-
/usr/hdp/3.1.5.6091-7/hadoop/lib/native:/usr/jdk64/jdk1.8.0_112/jre:/usr/hdp/3.1.5.6091-7/usr/lib/:/usr/hdp/3.1.5.6091-7/usr/lib/
LIVY_URL: 'http://xyz:8999'
MLFLOW_TRACKING_URI: 'http://mlflow:5100'
NO_PROXY: mlflow
SPARK_CONF_DIR: /etc/spark2/conf
SPARK_HOME: /usr/hdp/3.1.5.6091-7/spark2
SPARK2_PARCEL_DIR: /usr/hdp/3.1.5.6091-7/spark2
TOOLS_BASE_PATH: /usr/local/bin
image: >-
/jupyterhub/jpt-spark-magic:1.1.2
mem_limit: 8192M
uid: 0
- description: Python for data enthusiasts test3
display_name: 3
kubespawner_override:
cmd:
- jpt-entry-cmd.sh
cpu_limit: 8
environment:
XYZ_SERVICE_URL: 'http://XYZ-service:8080'
CURL_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
DOCKER_HOST: 'tcp://localhost:2375'
HADOOP_CONF_DIR: /etc/hadoop/conf
HADOOP_HOME: /usr/hdp/3.1.5.6091-7/hadoop/
HDP_DIR: /usr/hdp/3.1.5.6091-7
HDP_HOME_DIR: /usr/hdp/3.1.5.6091-7
HDP_VERSION: 3.1.5.6091-7
HIVE_CONF_DIR: /usr/hdp/3.1.5.6091-7/hive
HIVE_HOME: /usr/hdp/3.1.5.6091-7/hive
INTEGRATION_ENV: HDP3
JAVA_HOME: /usr/jdk64/jdk1.8.0_112
LD_LIBRARY_PATH: >-
/usr/hdp/3.1.5.6091-7/hadoop/lib/native:/usr/jdk64/jdk1.8.0_112/jre:/usr/hdp/3.1.5.6091-7/usr/lib/:/usr/hdp/3.1.5.6091-7/usr/lib/
LIVY_URL: 'http://fake.org:8999'
MLFLOW_TRACKING_URI: 'http://mlflow:5100'
NO_PROXY: mlflow
SPARK_CONF_DIR: /etc/spark2/conf
SPARK_HOME: /usr/hdp/3.1.5.6091-7/spark2
SPARK2_PARCEL_DIR: /usr/hdp/3.1.5.6091-7/spark2
TOOLS_BASE_PATH: /usr/local/bin
image: >-
/jupyterhub/jpt-spark-magic:1.1.2
mem_limit: 16384M
uid: 0
startTimeout: 300
storage:
capacity: 10Gi
dynamic:
pvcNameTemplate: 'claim-{username}{servername}'
storageAccessModes:
- ReadWriteOnce
storageClass: nfs-client
volumeNameTemplate: 'volume-{username}{servername}'
extraLabels: {}
extraVolumeMounts:
- mountPath: /etc/krb5.conf
name: krb
readOnly: true
- mountPath: /usr/jdk64/jdk1.8.0_112
name: java-home
readOnly: true
- mountPath: /xyz/conda/envs
name: xyz-conda-envs
readOnly: false
- mountPath: /usr/hdp/
name: bigdata
readOnly: true
subPath: usr-hdp
- mountPath: /etc/hadoop/
name: bigdata
readOnly: true
subPath: HDP
- mountPath: /etc/hive/
name: bigdata
readOnly: true
subPath: hdp-hive
- mountPath: /etc/spark2/
name: bigdata
readOnly: true
subPath: hdp-spark2
extraVolumes:
- emptyDir: {}
name: dind-storage
- name: docker-cert
secret:
secretName: docker-cert
- hostPath:
path: /var/lib/ut_xyz_ts/jdk1.8.0_112
type: Directory
name: java-home
- hostPath:
path: /xyz/conda/envs
type: Directory
name: xyz-conda-envs
- hostPath:
path: /etc/krb5.conf
type: File
name: krb
- name: bigdata
persistentVolumeClaim:
claimName: bigdata
homeMountPath: '/home/{username}'
static:
subPath: '{username}'
type: dynamic
uid: 0
Thanks in advance ..
For you question, whether technically two or more containers of same pod share same volume, the answer in Yes. Refer here - https://youtu.be/GQJP9QdHHs8?t=82 .
But you should have a volumeMount (refer example in the video as well) defined in your extra containers spec as well. If you can check that, or share output of kubectl describe deployment <your-deployment> I can confirm it.
Background: I have setup a ServiceAccount and spinnaker-role-binding in the default namespace. Created the spinnaker namespace for Kubernetes. Deployed services on port 9000 and 8084.
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/spin-deck-np LoadBalancer hidden <pending> 9000:31295/TCP 9m39s
service/spin-gate-np LoadBalancer hidden <pending> 8084:32161/TCP 9m39s
Created halyard deployment in the default namespace and configured hal inside it.
Problem: When I run the hal deploy apply command then I am getting below error
Problems in Global:
! ERROR Unexpected exception:
io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: GET
at:
https://kubernetes.default/apis/extensions/v1beta1/namespaces/spinnaker/replicasets.
Message: the server could not find the requested resource. Received status:
Status(apiVersion=v1, code=404, details=StatusDetails(causes=[], group=null,
kind=null, name=null, retryAfterSeconds=null, uid=null,
additionalProperties={}), kind=Status, message=the server could not find the
requested resource, metadata=ListMeta(resourceVersion=null, selfLink=null,
additionalProperties={}), reason=NotFound, status=Failure,
additionalProperties={}).
Below is my kube config file at /home/spinnaker/.kube/config
apiVersion: v1
clusters:
- cluster:
certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
server: https://kubernetes.default
name: default
contexts:
- context:
cluster: default
user: user
name: default
current-context: default
kind: Config
preferences: {}
users:
- name: user
user:
token: *********************
Below is the hal config file at /home/spinnaker/.hal/config
currentDeployment: default
deploymentConfigurations:
- name: default
version: 1.8.1
providers:
appengine:
enabled: false
accounts: []
aws:
enabled: false
accounts: []
bakeryDefaults:
baseImages: []
defaultKeyPairTemplate: '{{name}}-keypair'
defaultRegions:
- name: us-west-2
defaults:
iamRole: BaseIAMRole
ecs:
enabled: false
accounts: []
azure:
enabled: false
accounts: []
bakeryDefaults:
templateFile: azure-linux.json
baseImages: []
dcos:
enabled: false
accounts: []
clusters: []
dockerRegistry:
enabled: true
accounts:
- name: my-docker-registry
requiredGroupMembership: []
providerVersion: V1
permissions: {}
address: https://index.docker.io
email: fake.email#spinnaker.io
cacheIntervalSeconds: 30
clientTimeoutMillis: 60000
cacheThreads: 1
paginateSize: 100
sortTagsByDate: false
trackDigests: false
insecureRegistry: false
repositories:
- library/nginx
primaryAccount: my-docker-registry
google:
enabled: false
accounts: []
bakeryDefaults:
templateFile: gce.json
baseImages: []
zone: us-central1-f
network: default
useInternalIp: false
kubernetes:
enabled: true
accounts:
- name: my-k8s-account
requiredGroupMembership: []
providerVersion: V1
permissions: {}
dockerRegistries:
- accountName: my-docker-registry
namespaces: []
configureImagePullSecrets: true
cacheThreads: 1
namespaces: []
omitNamespaces: []
kinds: []
omitKinds: []
customResources: []
cachingPolicies: []
kubeconfigFile: /home/spinnaker/.kube/config
oauthScopes: []
oAuthScopes: []
primaryAccount: my-k8s-account
openstack:
enabled: false
accounts: []
bakeryDefaults:
baseImages: []
oracle:
enabled: false
accounts: []
deploymentEnvironment:
size: SMALL
type: Distributed
accountName: my-k8s-account
updateVersions: true
consul:
enabled: false
vault:
enabled: false
customSizing: {}
gitConfig:
upstreamUser: spinnaker
persistentStorage:
persistentStoreType: gcs
azs: {}
gcs:
jsonPath: /home/spinnaker/.gcp/gcs-account.json
project: round-reality
bucket: spin-94cc2e22-8ece-4bc1-80fd-e9df71c1d9f4
rootFolder: front50
bucketLocation: us
redis: {}
s3:
rootFolder: front50
oracle: {}
features:
auth: false
fiat: false
chaos: false
entityTags: false
jobs: false
metricStores:
datadog:
enabled: false
prometheus:
enabled: false
add_source_metalabels: true
stackdriver:
enabled: false
period: 30
enabled: false
notifications:
slack:
enabled: false
timezone: America/Los_Angeles
ci:
jenkins:
enabled: false
masters: []
travis:
enabled: false
masters: []
security:
apiSecurity:
ssl:
enabled: false
overrideBaseUrl: http://External IP of worker:8084
uiSecurity:
ssl:
enabled: false
overrideBaseUrl: http://External IP of worker:9000
authn:
oauth2:
enabled: false
client: {}
resource: {}
userInfoMapping: {}
saml:
enabled: false
ldap:
enabled: false
x509:
enabled: false
iap:
enabled: false
enabled: false
authz:
groupMembership:
service: EXTERNAL
google:
roleProviderType: GOOGLE
github:
roleProviderType: GITHUB
file:
roleProviderType: FILE
enabled: false
artifacts:
bitbucket:
enabled: false
accounts: []
gcs:
enabled: false
accounts: []
github:
enabled: false
accounts: []
gitlab:
enabled: false
accounts: []
http:
enabled: false
accounts: []
s3:
enabled: false
accounts: []
pubsub:
google:
enabled: false
subscriptions: []
canary:
enabled: false
serviceIntegrations:
- name: google
enabled: false
accounts: []
gcsEnabled: false
stackdriverEnabled: false
- name: prometheus
enabled: false
accounts: []
- name: datadog
enabled: false
accounts: []
- name: aws
enabled: false
accounts: []
s3Enabled: false
reduxLoggerEnabled: true
defaultJudge: NetflixACAJudge-v1.0
stagesEnabled: true
templatesEnabled: true
showAllConfigsEnabled: true
Used below commands in hal to interact with kubernetes
kubectl config set-cluster default --server=https://kubernetes.default --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
kubectl config set-context default --cluster=default
token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
kubectl config set-credentials user --token=$token
kubectl config set-context default --user=user
kubectl config use-context default
How could I resolve the error for spinnaker deployment?
Thank you
As per your config file it's looking like kubeconfig context(Search it) not setup correctly.
Please use below command
# Setting Variable for admin kubeconfig file location(Please fetch config file with --admin - if possible)
kubeconfig_path="<my-k8s-account-admin-file-path>"
hal config provider kubernetes account add my-k8s-account --provider-version v2 \
--kubeconfig-file "$kubeconfig_path" \
--context $(kubectl config current-context --kubeconfig "$kubeconfig_path")
After execution of above command you will be able to see context in your config file, which is missing in current config.
I've configured spinnaker cloud provider as kubernetes with below commands
hal config provider kubernetes enable
kubectl config current-context
CONTEXT=$(kubectl config current-context)
hal config provider kubernetes account add my-k8s-v2-account --provider-version v2 --context $CONTEXT
hal config features edit --artifacts true
but this account is not visible on spinnaker UI
and in logs its shows error as below
Nov 29 12:07:43 47184UW2DDevLVM2 gate[34594]: 2019-11-29 12:07:43.860 ERROR 34594 --- [TaskScheduler-5] c.n.s.g.s.DefaultProviderLookupService : Unable to refresh account details cache, reason: timeout
please advise.. thanks..
here's my hal deploy diff command output
+ Get current deployment
Success
+ Determine config diff
Success
~ EDITED
default.persistentStorage.redis
- port 6379 -> null
- host localhost -> null
~ EDITED
telemetry
I've provisioned new VM and did all installation process from scratch but still same issue :(
here is ~/.kube/config file
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: xxx
server: https://xxx:443
name:xxx
contexts:
- context:
cluster: xxx
user: xxx
name: xxx
current-context: xxx
kind: Config
preferences: {}
users:
- name: xxx
user:
client-certificate-data: xxx
client-key-data: xxx
token: xxx
and here is ~/.hal/config file
currentDeployment: default
deploymentConfigurations:
- name: default
version: 1.17.2
providers:
appengine:
enabled: false
accounts: []
aws:
enabled: false
accounts: []
bakeryDefaults:
baseImages: []
defaultKeyPairTemplate: '{{name}}-keypair'
defaultRegions:
- name: xxx
defaults:
iamRole: BaseIAMRole
ecs:
enabled: false
accounts: []
azure:
enabled: false
accounts: []
bakeryDefaults:
templateFile: azure-linux.json
baseImages: []
dcos:
enabled: false
accounts: []
clusters: []
dockerRegistry:
enabled: false
accounts: []
google:
enabled: false
accounts: []
bakeryDefaults:
templateFile: gce.json
baseImages: []
zone: us-central1-f
network: default
useInternalIp: false
kubernetes:
enabled: true
accounts:
- name: xxx
requiredGroupMembership: []
providerVersion: V2
permissions: {}
dockerRegistries: []
context: xxx
configureImagePullSecrets: true
cacheThreads: 1
namespaces: []
omitNamespaces: []
kinds: []
omitKinds: []
customResources: []
cachingPolicies: []
kubeconfigFile: /home/xxx/.kube/config
oAuthScopes: []
onlySpinnakerManaged: false
primaryAccount: xxx
oracle:
enabled: false
accounts: []
bakeryDefaults:
templateFile: oci.json
baseImages: []
cloudfoundry:
enabled: false
accounts: []
deploymentEnvironment:
size: SMALL
type: LocalDebian
imageVariant: SLIM
updateVersions: true
consul:
enabled: false
vault:
enabled: false
customSizing: {}
sidecars: {}
initContainers: {}
hostAliases: {}
affinity: {}
tolerations: {}
nodeSelectors: {}
gitConfig:
upstreamUser: spinnaker
livenessProbeConfig:
enabled: false
haServices:
clouddriver:
enabled: false
disableClouddriverRoDeck: false
echo:
enabled: false
persistentStorage:
persistentStoreType: azs
azs:
storageAccountName: xxx
storageAccountKey: xxx
storageContainerName: xxx
gcs:
rootFolder: front50
redis: {}
s3:
rootFolder: front50
oracle: {}
features:
auth: false
fiat: false
chaos: false
entityTags: false
artifacts: true
metricStores:
datadog:
enabled: false
tags: []
prometheus:
enabled: false
add_source_metalabels: true
stackdriver:
enabled: false
newrelic:
enabled: false
tags: []
period: 30
enabled: false
notifications:
slack:
enabled: false
twilio:
enabled: false
baseUrl: https://api.twilio.com/
github-status:
enabled: false
timezone: America/Los_Angeles
ci:
jenkins:
enabled: false
masters: []
travis:
enabled: false
masters: []
wercker:
enabled: false
masters: []
concourse:
enabled: false
masters: []
gcb:
enabled: false
accounts: []
repository:
artifactory:
enabled: false
searches: []
security:
apiSecurity:
ssl:
enabled: false
overrideBaseUrl: http://xxx:8084/
uiSecurity:
ssl:
enabled: false
overrideBaseUrl: http://xxx:9000/
authn:
oauth2:
enabled: false
client: {}
resource: {}
userInfoMapping: {}
saml:
enabled: false
userAttributeMapping: {}
ldap:
enabled: false
x509:
enabled: false
iap:
enabled: false
enabled: false
authz:
groupMembership:
service: EXTERNAL
google:
roleProviderType: GOOGLE
github:
roleProviderType: GITHUB
file:
roleProviderType: FILE
ldap:
roleProviderType: LDAP
enabled: false
artifacts:
bitbucket:
enabled: false
accounts: []
gcs:
enabled: false
accounts: []
oracle:
enabled: false
accounts: []
github:
enabled: false
accounts: []
gitlab:
enabled: false
accounts: []
gitrepo:
enabled: false
accounts: []
http:
enabled: false
accounts: []
helm:
enabled: false
accounts: []
s3:
enabled: false
accounts: []
maven:
enabled: false
accounts: []
templates: []
pubsub:
enabled: false
google:
enabled: false
pubsubType: GOOGLE
subscriptions: []
publishers: []
canary:
enabled: false
serviceIntegrations:
- name: google
enabled: false
accounts: []
gcsEnabled: false
stackdriverEnabled: false
- name: prometheus
enabled: false
accounts: []
- name: datadog
enabled: false
accounts: []
- name: signalfx
enabled: false
accounts: []
- name: aws
enabled: false
accounts: []
s3Enabled: false
- name: newrelic
enabled: false
accounts: []
reduxLoggerEnabled: true
defaultJudge: NetflixACAJudge-v1.0
stagesEnabled: true
templatesEnabled: true
showAllConfigsEnabled: true
plugins:
plugins: []
enabled: false
downloadingEnabled: false
pluginConfigurations:
plugins: {}
webhook:
trust:
enabled: false
telemetry:
enabled: false
endpoint: https://stats.spinnaker.io
instanceId: xxx
connectionTimeoutMillis: 3000
readTimeoutMillis: 5000
Here are the commands used to install spinnaker
az login
az aks get-credentials --resource-group xxx --name xxx
curl -O https://raw.githubusercontent.com/spinnaker/halyard/master/install/debian/InstallHalyard.sh
sudo bash InstallHalyard.sh --user xxx
hal config provider kubernetes enable
CONTEXT=$(kubectl config current-context)
hal config provider kubernetes account add xxx \
--provider-version v2 \
--context $CONTEXT
hal config features edit --artifacts true
hal config deploy edit --type localdebian
hal config storage azs edit --storage-account-name xxx --storage-account-key xxx
hal config storage edit --type azs
hal version list
hal config version edit --version 1.17.2
sudo hal deploy apply
echo "host: 0.0.0.0" | tee \
~/.hal/default/service-settings/gate.yml \
~/.hal/default/service-settings/deck.yml
hal config security ui edit \
--override-base-url http://xxx:9000/
hal config security api edit \
--override-base-url http://xxx:8084/
sudo hal deploy apply
Found below exceptions logs
Dec 2 11:12:07 47184UW2DDevLVM2 clouddriver[23908]: 2019-12-02 11:12:07.424 ERROR 23908 --- [1-7002-exec-105] c.n.s.k.w.e.GenericExceptionHandlers : Internal Server Error
Dec 2 11:12:07 47184UW2DDevLVM2 clouddriver[23908]: java.lang.NullPointerException: null
Dec 2 11:12:07 47184UW2DDevLVM2 clouddriver[23908]: #011at com.netflix.spinnaker.clouddriver.kubernetes.health.KubernetesHealthIndicator.health(KubernetesHealthIndicator.java:48) ~[clouddriver-kubernetes-6.4.1-20191111102213.jar:6.4.1-20191111102213]
Dec 2 11:12:07 47184UW2DDevLVM2 clouddriver[23908]: #011at org.springframework.boot.actuate.health.CompositeHealthIndicator.health(CompositeHealthIndicator.java:95) ~[spring-boot-actuator-2.1.7.RELEASE.jar:2.1.7.RELEASE]
Dec 2 11:12:07 47184UW2DDevLVM2 clouddriver[23908]: #011at org.springframework.boot.actuate.health.HealthEndpoint.health(HealthEndpoint.java:50) ~[spring-boot-actuator-2.1.7.RELEASE.jar:2.1.7.RELEASE]
Dec 2 11:12:07 47184UW2DDevLVM2 clouddriver[23908]: #011at org.springframework.boot.actuate.health.HealthEndpointWebExtension.health(HealthEndpointWebExtension.java:53) ~[spring-boot-actuator-2.1.7.RELEASE.jar:2.1.7.RELEASE]
plus localhost 7002 is not responding
hexunix#47184UW2DDevLVM2:~$ curl -v http://localhost:7002/credentials
* Trying 127.0.0.1...
* TCP_NODELAY set
* Connected to localhost (127.0.0.1) port 7002 (#0)
> GET /credentials HTTP/1.1
> Host: localhost:7002
> User-Agent: curl/7.58.0
> Accept: */*
>
This is how i have done in my environment
kubeconfig_path="/home/root/.hal/kube-config"
kubernetes_account="my-account"
docker_registry="docker.io"
hal config provider kubernetes account add $kubernetes_account --provider-version v2 \
--kubeconfig-file "$kubeconfig_path" \
--context $(kubectl config current-context --kubeconfig "$kubeconfig_path") \
--omit-namespaces=kube-system,kube-public \
--docker-registries "$docker_registry"
make necessary updates and apply the changes. It should work.
from hal config it is clear that kubernetes account is added.
kubernetes:
enabled: true
accounts:
- name: xxx
requiredGroupMembership: []
providerVersion: V2
permissions: {}
dockerRegistries: []
context: xxx
configureImagePullSecrets: true
cacheThreads: 1
namespaces: []
omitNamespaces: []
kinds: []
omitKinds: []
customResources: []
cachingPolicies: []
kubeconfigFile: /home/xxx/.kube/config
oAuthScopes: []
onlySpinnakerManaged: false
primaryAccount: xxx
Using kubeadm init initializes the control plane with default configuration options.
Is there a way to see what default values/configuration it will use for the control plane, how can I view that configuration file, and where is it stored?
Found the command: ( just in case someone needs it)
C02W84XMHTD5:~ iahmad$ kubectl get configMap kubeadm-config -o yaml --namespace=kube-system
apiVersion: v1
data:
MasterConfiguration: |
api:
advertiseAddress: 192.168.64.4
bindPort: 8443
controlPlaneEndpoint: localhost
apiServerExtraArgs:
admission-control: Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota
auditPolicy:
logDir: /var/log/kubernetes/audit
logMaxAge: 2
path: ""
authorizationModes:
- Node
- RBAC
certificatesDir: /var/lib/minikube/certs/
cloudProvider: ""
criSocket: /var/run/dockershim.sock
etcd:
caFile: ""
certFile: ""
dataDir: /data/minikube
endpoints: null
image: ""
keyFile: ""
imageRepository: k8s.gcr.io
kubeProxy:
config:
bindAddress: 0.0.0.0
clientConnection:
acceptContentTypes: ""
burst: 10
contentType: application/vnd.kubernetes.protobuf
kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
qps: 5
clusterCIDR: ""
configSyncPeriod: 15m0s
conntrack:
max: null
maxPerCore: 32768
min: 131072
tcpCloseWaitTimeout: 1h0m0s
tcpEstablishedTimeout: 24h0m0s
enableProfiling: false
healthzBindAddress: 0.0.0.0:10256
hostnameOverride: ""
iptables:
masqueradeAll: false
masqueradeBit: 14
minSyncPeriod: 0s
syncPeriod: 30s
ipvs:
minSyncPeriod: 0s
scheduler: ""
syncPeriod: 30s
metricsBindAddress: 127.0.0.1:10249
mode: ""
nodePortAddresses: null
oomScoreAdj: -999
portRange: ""
resourceContainer: /kube-proxy
udpIdleTimeout: 250ms
kubeletConfiguration: {}
kubernetesVersion: v1.10.0
networking:
dnsDomain: cluster.local
podSubnet: ""
serviceSubnet: 10.96.0.0/12
noTaintMaster: true
nodeName: minikube
privilegedPods: false
token: ""
tokenGroups:
- system:bootstrappers:kubeadm:default-node-token
tokenTTL: 24h0m0s
tokenUsages:
- signing
- authentication
unifiedControlPlaneImage: ""
kubeadm config print init-defaults (cf. join-defaults) will show the defaults:
$ kubeadm config print init-defaults
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 1.2.3.4
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: k1.london.vultr
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.16.0
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/12
scheduler: {}
(kubeadm v1.16.0)
As far as I can tell these are built-in defaults, and there is no overridable defaults file, i.e. to supply a different config file it is necessary to pass --config=<file>.