Having problem in authenticating kubernetes python client - kubernetes

my lisNamespaces.py file
from __future__ import print_function
import time
import kubernetes.client
from kubernetes.client.rest import ApiException
configuration = kubernetes.client.Configuration()
configuration.ssl_ca_cert = 'LS0XXXXXXXXXS0tLQo='
configuration.api_key['authorization'] = 'ZXXXXXXXXXXdw=='
configuration.api_key_prefix['authorization'] = 'Bearer'
configuration.host = 'https://aaaaaaaaaaaaaaa.gr7.us-east-1.eks.amazonaws.com'
#configuration.verify_ssl = False
api_instance = kubernetes.client.CoreV1Api(kubernetes.client.ApiClient(configuration))
api_response = api_instance.list_namespace()
for i in api_response.items:
print(i.metadata.name)
For ssl_ca_cert value i did kubectl edit secret nameofsa-token-xyze -n default and used ca.crt value. user has cluster level admin permissions
For bearer token i have used same user TOKEN.
If i disable ssl verification by setting configuration.verify_ssl = False my code works fine but with an warining.
i want to know what mistake i am doing here in passing ssl_ca_cert. please help me with this.

Mistake i did was to pass data of ca.crt which i got from kubectl edit secret nameofsa-token-xyze -n default directly to configuration.ssl_ca_cert in the code.
Instead what should be done is to decode the data using base64 --decode, which i got from above command(kubectl edit secret nameofsa-token-xyze -n default), this is how i did it.
kubectl get secrets default-token-nqkdv -n default -o jsonpath='{.data.ca\.crt}' | base64 --decode > ca.crt.
Then i need to pass the path of ca.crt file in the code, so final code look like below
from __future__ import print_function
import time
import kubernetes.client
from kubernetes.client.rest import ApiException
configuration = kubernetes.client.Configuration()
configuration.ssl_ca_cert = 'ca.crt'
configuration.api_key['authorization'] = 'ZXXXXXXXXXXdw=='
configuration.api_key_prefix['authorization'] = 'Bearer'
configuration.host = 'https://aaaaaaaaaaaaaaa.gr7.us-east-1.eks.amazonaws.com'
api_instance = kubernetes.client.CoreV1Api(kubernetes.client.ApiClient(configuration))
api_response = api_instance.list_namespace()
for i in api_response.items:
print(i.metadata.name)

You can test the token with basic request:
import requests
with open('/path/to/token', 'r') as token_file:
token=token_file.read()
url = 'https://my-kubernetes-cluster'
headers = {"Authorization":"Bearer "+token}
r = requests.get(url, verify='/path/to/ca_chain.crt', headers=headers)
for line in r.iter_lines():
print line
If the request goes through you can test this code:
from kubernetes import client
from kubernetes.client import Configuration, ApiClient
config = Configuration()
config.api_key = {'authorization': 'Bearer <api_key>'}
config.host = 'https://my-kubernetes-cluster'
config.ssl_ca_cert = "/path/to/ca_chain.crt"
api_client = ApiClient(configuration=config)
v1 = client.CoreV1Api(api_client)
v1.list_pod_for_all_namespaces(watch=False)
Try and let me know if it works for you.

Related

Get list of all notebooks in my databricks workspace

How do I get a list of all notebooks in my workspace & store their names along with full path in csv file, I have tried using Databricks CLI option but that doesn't seem to have recursive operation.
databricks workspace list
As we can see in code there is no recursive option:
https://github.com/databricks/databricks-cli/blob/master/databricks_cli/workspace/cli.py (def ls_cli)
Example solution is to import cli in python and extend it:
from databricks_cli.sdk import ApiClient
from databricks_cli.sdk import service
host = "your_host"
token = "your_token"
client = ApiClient(host=host, token=token)
objects = []
workspace = service.WorkspaceService(client)
def list_workspace_objects(path):
elements = workspace.list(path).get('objects')
if elements is not None:
for object in elements:
objects.append(object)
if(object['object_type'] == 'DIRECTORY'):
list_workspace_objects(object['path'])
list_workspace_objects("/")
print(objects)
You can use below code directly . Note : Tested Code
from pyspark.sql.types import IntegerType
from pyspark.sql.types import *
from pyspark.sql import Row
import base64
import requests
import json
databricks_instance ="databricks Instance"
url_list = f"{databricks_instance}/api/2.0/workspace/list"
url_export = f"{databricks_instance}/api/2.0/workspace/export"
payload = json.dumps({
"path": "/"
})
headers = {
'Authorization': 'Bearer token',
'Content-Type': 'application/json'
}
response = requests.request("GET", url_list, headers=headers, data=payload).json()
notebooks = []
# Getting the all notebooks list for given notebooks.
def list_notebooks(mylist):
for element in mylist['objects']:
if element['object_type'] == 'NOTEBOOK':
notebooks.append(element)
if element['object_type'] == 'DIRECTORY':
payload_inner = json.dumps({
"path": element['path']
})
response_inner = requests.request("GET", url_list, headers=headers, data=payload_inner).json()
if len(response_inner) != 0:
list_notebooks(response_inner)
return notebooks
result = list_notebooks(response)
print(result[0])

python 3.10 soap.find(id='productTitle').get_text(strip=True) NoneType Error

soap.find(id='productTitle').get_text(strip=True)
Output: 'NoneType' Object has no attribute 'get_text'.
There's not a lot to go off since you didn't provide a lot of information, but from the information I got, you've put soap.find instead of soup.find
You could try something like this to fix it:
import requests
from bs4 import BeautifulSoup
URL = "Your url"
headers = {
"User-Agent": '(search My user agent)'}
def product_title():
req = requests.Session()
page = req.get(URL, headers=headers)
soup = BeautifulSoup(page.content, 'html.parser')
productTitle = soup.find(id='productTitle').get_text(strip=True)
print(product)
productTitle()

"Dag Seems to be missing" error in a Cloud Composer Airflow Dynamic DAG

I have a dynamic Airflow DAG in Google Cloud Composer gets created, listed in the web-server and ran (backfill) without error.
However, there are issues:
When clicking on the DAG in web url, it says "DAG seems to be
missing"
Can't see Graph view/Tree view as showing the error above
Can't manually trigger the DAG as showing the error above
Trying to fix this for couple days...any hint will be helpful. Thank you!
from airflow import DAG
from airflow.operators.dummy_operator import DummyOperator
from airflow.operators.python_operator import PythonOperator
from airflow.contrib.operators.gcs_to_bq import GoogleCloudStorageToBigQueryOperator
from google.cloud import storage
from airflow.models import Variable
import json
args = {
'owner': 'xxx',
'start_date':'2020-11-5',
'provide_context': True
}
dag = DAG(
dag_id='dynamic',
default_args=args
)
def return_bucket_files(bucket_name='xxxxx', **kwargs):
client = storage.Client()
bucket = client.get_bucket(bucket_name)
blobs = bucket.list_blobs()
file_list = [blob.name for blob in blobs]
return file_list
def dynamic_gcs_to_gbq_etl(file, **kwargs):
mapping = json.loads(Variable.get("xxxxx"))
database = mapping[0][file]
table = mapping[1][file]
task=GoogleCloudStorageToBigQueryOperator(
task_id= f'gcs_load_{file}_to_gbq',
bucket='xxxxxxx',
source_objects=[f'{file}'],
destination_project_dataset_table=f'xxx.{database}.{table}',
write_disposition="WRITE_TRUNCATE",
autodetect=True,
skip_leading_rows=1,
source_format='CSV',
dag=dag)
return task
start_task = DummyOperator(
task_id='start',
dag=dag
)
end_task = DummyOperator(
task_id='end',
dag=dag)
push_bucket_files = PythonOperator(
task_id="return_bucket_files",
provide_context=True,
python_callable=return_bucket_files,
dag=dag)
for file in return_bucket_files():
gcs_load_task = dynamic_gcs_to_gbq_etl(file)
start_task >> push_bucket_files >> gcs_load_task >> end_task
This issue means that the Web Server is failing to fill in the DAG bag on its side - this problem is most likely not with your DAG specifically.
My suggestion would be right now to try and restart the web server (via the installation of some dummy package).
Similar issues reported in this post as well here.

clients = self.AVAILABLE_CLIENTS[name] KeyError: 'requests' flask authlib client

good day everybody,
having some issues with flask and authlib. Bellow snip of my flash code
from flask import Flask, render_template
from authlib.integrations.flask_client import OAuth
import os
app = Flask(__name__)
app._static_folder = os.path.abspath("static")
app.config.from_object('config')
oauth = OAuth(app)
webex = oauth.register(name='WEBEX', redirect_uri='http://webapp.dcloud.cisco.com:5000/AuthWebex', client_kwargs={
'scope': 'spark:all'
} )
config.py
import os
WEBEX_CLIENT_ID='C3a256be511cdf07e19f272960c44a214aec14b727b108e4f10bd124d31d2112c'
WEBEX_CLIENT_SECRET='secret'
WEBEX_ACCESS_TOKEN_URL='https://api.ciscospark.com/v1/access_token'
WEBEX_REDIRECT_URI='http://localhost:5000/AuthWebex'
WEBEX_SCOPE='spark:all'
when running above code I get the following error:
File "/Users/tneumann/PycharmProjects/untitled/venv/lib/python3.7/site-packages/authlib/integrations/flask_client/oauth_registry.py", line 61, in register
self.use_oauth_clients()
File "/Users/tneumann/PycharmProjects/untitled/venv/lib/python3.7/site-packages/authlib/integrations/_client/oauth_registry.py", line 49, in use_oauth_clients
clients = self.AVAILABLE_CLIENTS[name]
KeyError: 'requests'
looked at examples and did some research, no luck. Can't find any solution...
thanks in adv.
Tobi
UPDATE:
per comment bellow here the latest code:
from flask import Flask, render_template, url_for, request
from authlib.integrations.flask_client import OAuth
import os
import requests
app = Flask(__name__)
app._static_folder = os.path.abspath("static")
app.config.from_object('config')
app.secret_key = os.urandom(24)
oauth = OAuth(app)
oauth.register(
'webex',
api_base_url='https://api.ciscospark.com/v1',
authorize_url='https://api.ciscospark.com/v1/authorize',
access_token_url='https://api.ciscospark.com/v1/access_token',
redirect_uri='http://webapp.dcloud.cisco.com:5000/AuthWebex',
scope='spark:all')
#app.route('/')
def main():
"""Entry point; the view for the main page"""
return render_template('main.html')
#app.route('/authorize')
def authorize():
return render_template('authorize.html')
#app.route('/login')
def login():
#redirect_uri = url_for('AuthWebex', _external=True)
redirect_uri = 'http://webapp.dcloud.cisco.com:5000/AuthWebex'
print(redirect_uri)
return oauth.webex.authorize_redirect(redirect_uri)
#app.route('/AuthWebex')
def AuthWebex():
#print(request.__dict__)
token = oauth.webex.authorize_access_token( authorization_response=request.url,
redirect_uri='http://webapp.dcloud.cisco.com:5000/AuthWebex',
client_id='C3a256be511cdf07e19f272960c44a214aec14b727b108e4f10bd124d31d2112c',
client_secret='secret',
)
print("Token: ", token)
resp = oauth.webex.get('https://api.ciscospark.com/v1/people/me')
profile = resp.json()
print(profile)
# do something with the token and profile
return '<html><body><h1>Authenticated</h1></body></html>'
if __name__ == '__main__':
app.run()
oauth.webex.authorize_access_token function throws and error when called without the parameters. which is strange as most examples I found exactly do that.
client_id and client_secret are set via the config.py file. This works for the oauth.register function but not for the authorize_access_token.
Additional problem is that even with the parameters, it produces a valid token. When I call the get function I get the following error:
File "/Users/tneumann/PycharmProjects/untitled/venv/lib/python3.7/site-packages/requests/models.py", line 317, in prepare
self.prepare_auth(auth, url)
File "/Users/tneumann/PycharmProjects/untitled/venv/lib/python3.7/site-packages/requests/models.py", line 548, in prepare_auth
r = auth(self)
File "/Users/tneumann/PycharmProjects/untitled/venv/lib/python3.7/site-packages/authlib/integrations/requests_client/oauth2_session.py", line 41, in __call__
raise UnsupportedTokenTypeError(description=description)
authlib.integrations._client.errors.UnsupportedTokenTypeError: unsupported_token_type: Unsupported token_type: 'token_type'
here is the format of the token returned from authorize_access_token function
Token: {'access_token': 'YWIzNGU3<secret>tNDQ5_PF84_7cc07dbd-<secret>-5877334424fd', 'expires_in': 1209599, 'refresh_token': 'MjU2ZDM4N2Et<secret>ZmItMTg5_PF84_7cc07dbd-<secret>877334424fd', 'refresh_token_expires_in': 7722014, 'expires_at': 1574863645}
went through the docs, the code on github and debugging in pycharm with no luck, help would be much appreciated!
The problem here is that this AuthWebex is not a standard OAuth service. The response has no token_type. We can fix it with Authlib compliance fix:
Check the example here:
https://docs.authlib.org/en/latest/client/frameworks.html#compliance-fix-for-oauth-2-0
The slack example has the same issue.

programmatically export grafana dashboard data

I have a visual in grafana. I can manually go to the menu click export and export the time series data in json. This works great. Is there a way I can script that in python?. Is there some api I can hit that will return the json of a visual?
I was googling around and it looks like I can use the api to create dashboards/visuals and administer them but not sure where how to use the api to export the data.
Here's a Python script to export then dashboard json, not the presented data. Tested on Python 2.7:
#!/usr/bin/env python
"""Grafana dashboard exporter"""
import json
import os
import requests
HOST = 'http://localhost:3000'
API_KEY = os.environ["grafana_api_key"]
DIR = 'exported-dashboards/'
def main():
headers = {'Authorization': 'Bearer %s' % (API_KEY,)}
response = requests.get('%s/api/search?query=&' % (HOST,), headers=headers)
response.raise_for_status()
dashboards = response.json()
if not os.path.exists(DIR):
os.makedirs(DIR)
for d in dashboards:
print ("Saving: " + d['title'])
response = requests.get('%s/api/dashboards/%s' % (HOST, d['uri']), headers=headers)
data = response.json()['dashboard']
dash = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': '))
name = data['title'].replace(' ', '_').replace('/', '_').replace(':', '').replace('[', '').replace(']', '')
tmp = open(DIR + name + '.json', 'w')
tmp.write(dash)
tmp.write('\n')
tmp.close()
if __name__ == '__main__':
main()
Usage:
You should first create an API key in Grafana and then run:
grafana_api_key=my-key python export-dash.py
Credit: This is a simplified version of https://github.com/percona/grafana-dashboards/blob/master/misc/export-dash.py
http://docs.grafana.org/http_api/data_source/#data-source-proxy-calls.
Visit your browser console (network tab) and you will see how it works there.
You could also use this Go client https://github.com/netsage-project/grafana-dashboard-manager
Its purpose is not what you are looking for, but it is possible to reuse that code.