I want to use the STS service to generate temporary credentials for use by third-party clients.
I configured STS lite based on the documentation.
This is my configuration file:
[global]
fsid = 42a7cae1-84d1-423e-93f4-04b0736c14aa
mon_initial_members = admin, node1, node2, node3
mon_host = 192.168.199.81,192.168.199.82,192.168.199.83,192.168.199.84
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
osd pool default size = 2
[client.rgw.admin]
rgw sts key = "1234567890"
rgw s3 auth use sts = true
When I execute the getSessionToken method, return a 405 error:
<Error>
<Code>MethodNotAllowed</Code>
<RequestId>tx000000000000000000003-005c73aed8-5e48-default</RequestId>
<HostId>5e48-default-default</HostId>
</Error>
This is my test code:
import os
import sys
import traceback
import boto3
from boto.s3.connection import S3Connection
from boto.sts import STSConnection
try:
host = 'http://192.168.199.81:7480'
access_key = '2324YFZ7QDEOSRL18QHR'
secret_key = 'rL9FabxCOw5LDbrHtmykiGSCjzpKLmEs9WPiNjVJ'
client = boto3.client('sts',
aws_access_key_id = access_key,
aws_secret_access_key = secret_key,
endpoint_url = host)
response = client.get_session_token(DurationSeconds=999)
print response
except:
print traceback.format_exc()
Can anyone tell me if my configuration is incorrect or if the version I tested does not provide STS service?
These are the versions I tested:
Ceph version 12.2.11 (26dc3775efc7bb286a1d6d66faee0ba30ea23eee) luminous (stable)
Ceph version 13.2.2 (02899bfda814146b021136e9d8e80eba494e1126) mimic (stable)
STS feature has been implemented since nautilus (14.2.x).
for more info: https://docs.ceph.com/docs/master/releases/nautilus/
Related
I have a connection problem with Cloud Sql Postgres from my Flask Rest API app.
I have a db.py file:
import os
from flask_sqlalchemy import SQLAlchemy
import sqlalchemy
db = SQLAlchemy()
def connect_unix_socket() -> sqlalchemy.engine.base.Engine:
""" Initializes a Unix socket connection pool for a Cloud SQL instance of Postgres. """
# Note: Saving credentials in environment variables is convenient, but not
# secure - consider a more secure solution such as
# Cloud Secret Manager (https://cloud.google.com/secret-manager) to help
# keep secrets safe.
db_user = os.environ["DB_USER"] # e.g. 'my-database-user'
db_pass = os.environ["DB_PASS"] # e.g. 'my-database-password'
db_name = os.environ["DB_NAME"] # e.g. 'my-database'
unix_socket_path = os.environ["INSTANCE_UNIX_SOCKET"] # e.g. '/cloudsql/project:region:instance'
pool = sqlalchemy.create_engine(
# Equivalent URL:
# postgresql+pg8000://<db_user>:<db_pass>#/<db_name>
# ?unix_sock=<INSTANCE_UNIX_SOCKET>/.s.PGSQL.5432
# Note: Some drivers require the `unix_sock` query parameter to use a different key.
# For example, 'psycopg2' uses the path set to `host` in order to connect successfully.
sqlalchemy.engine.url.URL.create(
drivername="postgresql+pg8000",
username=db_user,
password=db_pass,
database=db_name,
query={"unix_sock": "{}/.s.PGSQL.5432".format(unix_socket_path)},
),
# [START_EXCLUDE]
# Pool size is the maximum number of permanent connections to keep.
pool_size=5,
# Temporarily exceeds the set pool_size if no connections are available.
max_overflow=2,
# The total number of concurrent connections for your application will be
# a total of pool_size and max_overflow.
# 'pool_timeout' is the maximum number of seconds to wait when retrieving a
# new connection from the pool. After the specified amount of time, an
# exception will be thrown.
pool_timeout=30, # 30 seconds
# 'pool_recycle' is the maximum number of seconds a connection can persist.
# Connections that live longer than the specified amount of time will be
# re-established
pool_recycle=1800, # 30 minutes
# [END_EXCLUDE]
)
return pool
I import the db.py file in my app.py file:
import os
import sqlalchemy
from flask import Flask
from flask_smorest import Api
from flask_sqlalchemy import SQLAlchemy
from db import db, connect_unix_socket
import models
from resources.user import blp as UserBlueprint
# pylint: disable=C0103
app = Flask(__name__)
def init_connection_pool() -> sqlalchemy.engine.base.Engine:
# use a Unix socket when INSTANCE_UNIX_SOCKET (e.g. /cloudsql/project:region:instance) is defined
if unix_socket_path:
return connect_unix_socket()
raise ValueError(
"Missing database connection type. Please define one of INSTANCE_HOST, INSTANCE_UNIX_SOCKET, or INSTANCE_CONNECTION_NAME"
)
db = None
#app.before_first_request
def init_db() -> sqlalchemy.engine.base.Engine:
global db
db = init_connection_pool()
api = Api(app)
#app.route("/api")
def user_route():
return "Welcome user API!"
api.register_blueprint(UserBlueprint)
if __name__ == '__main__':
server_port = os.environ.get('PORT', '8080')
app.run(debug=True, port=server_port, host='0.0.0.0')
The app run correctly, when i call the end point to Get or Post users, the app crash and give me this error:
"The current Flask app is not registered with this 'SQLAlchemy'"
RuntimeError: The current Flask app is not registered with this 'SQLAlchemy' instance. Did you forget to call 'init_app', or did you create multiple 'SQLAlchemy' instances?
This is my User.py class:
from sqlalchemy.exc import SQLAlchemyError, IntegrityError
from db import db
from models import UserModel
from schemas import UserSchema
blp = Blueprint("Users", "users", description="Operations on users")
#blp.route("/user/<string:user_id>")
class User(MethodView):
#blp.response(200, UserSchema)
def get(self, user_id):
user = UserModel.query.get_or_404(user_id)
return user
def delete(self, user_id):
user = UserModel.query.get_or_404(user_id)
db.session.delete(user)
db.session.commit()
return {"message": "User deleted"}, 200
#blp.route("/user")
class UserList(MethodView):
#blp.response(200, UserSchema(many=True))
def get(self):
return UserModel.query.all()
How i can fix this issue?
#dev_ Your issue is that your are trying to intermingle the use of SQLAlchemy Core with SQLAlchemy ORM as if they are the same thing, leading to your issues. SQLAlchemy connection pools created using sqlalchemy.create_engine use the CORE API while Flask-SQLAlchemy uses the SQLAlchemy ORM model. This is the core reason for you issue. It is easier to use one or the other.
I would recommend using purely Flask-SQLALchemy with the use of the cloud-sql-python-connector library for your use-case. It will make your life much easier.
For simplicity, I am getting rid of your db.py leading to your app.py file being as follows:
from flask import Flask
from flask_smorest import Api
from flask_sqlalchemy import SQLAlchemy
from google.cloud.sql.connector import Connector, IPTypes
from resources.user import blp as UserBlueprint
# load env vars
db_user = os.environ["DB_USER"] # e.g. 'my-database-user'
db_pass = os.environ["DB_PASS"] # e.g. 'my-database-password'
db_name = os.environ["DB_NAME"] # e.g. 'my-database'
instance_connection_name = os.environ["INSTANCE_CONNECTION_NAME"] # e.g. 'project:region:instance'
# Python Connector database connection function
def getconn():
with Connector() as connector:
conn = connector.connect(
instance_connection_name, # Cloud SQL Instance Connection Name
"pg8000",
user=db_user,
password=db_pass,
db=db_name,
ip_type= IPTypes.PUBLIC # IPTypes.PRIVATE for private IP
)
return conn
app = Flask(__name__)
# configure Flask-SQLAlchemy to use Python Connector
app.config['SQLALCHEMY_DATABASE_URI'] = "postgresql+pg8000://"
app.config['SQLALCHEMY_ENGINE_OPTIONS'] = {
"creator": getconn
}
# initialize db (using app!)
db = SQLAlchemy(app)
# rest of your code
api = Api(app)
# ...
Hope this helps resolve your issue!
I'm have a Gitlab cloud connected to a k8s cluster running on Google (GKE).
The cluster was created via Gitlab cloud.
I want to customise the config.toml because I want to fix the cache on k8s as suggested in this issue.
I found the config.toml configuration in the runner-gitlab-runner ConfigMap.
I updated the ConfigMap to contain this config.toml setup:
config.toml: |
concurrent = 4
check_interval = 3
log_level = "info"
listen_address = '[::]:9252'
[[runners]]
executor = "kubernetes"
cache_dir = "/tmp/gitlab/cache"
[runners.kubernetes]
memory_limit = "1Gi"
[runners.kubernetes.node_selector]
gitlab = "true"
[[runners.kubernetes.volumes.host_path]]
name = "gitlab-cache"
mount_path = "/tmp/gitlab/cache"
host_path = "/home/core/data/gitlab-runner/data"
To apply the changes I deleted the runner-gitlab-runner-xxxx-xxx pod so a new one gets created with the updated config.toml.
However, when I look into the new pod, the /home/gitlab-runner/.gitlab-runner/config.toml now contains 2 [[runners]] sections:
listen_address = "[::]:9252"
concurrent = 4
check_interval = 3
log_level = "info"
[session_server]
session_timeout = 1800
[[runners]]
name = ""
url = ""
token = ""
executor = "kubernetes"
cache_dir = "/tmp/gitlab/cache"
[runners.kubernetes]
host = ""
bearer_token_overwrite_allowed = false
image = ""
namespace = ""
namespace_overwrite_allowed = ""
privileged = false
memory_limit = "1Gi"
service_account_overwrite_allowed = ""
pod_annotations_overwrite_allowed = ""
[runners.kubernetes.node_selector]
gitlab = "true"
[runners.kubernetes.volumes]
[[runners.kubernetes.volumes.host_path]]
name = "gitlab-cache"
mount_path = "/tmp/gitlab/cache"
host_path = "/home/core/data/gitlab-runner/data"
[[runners]]
name = "runner-gitlab-runner-xxx-xxx"
url = "https://gitlab.com/"
token = "<my-token>"
executor = "kubernetes"
[runners.cache]
[runners.cache.s3]
[runners.cache.gcs]
[runners.kubernetes]
host = ""
bearer_token_overwrite_allowed = false
image = "ubuntu:16.04"
namespace = "gitlab-managed-apps"
namespace_overwrite_allowed = ""
privileged = true
service_account_overwrite_allowed = ""
pod_annotations_overwrite_allowed = ""
[runners.kubernetes.volumes]
The file /scripts/config.toml is the configuration as I created it in the ConfigMap.
So I suspect the /home/gitlab-runner/.gitlab-runner/config.toml is somehow updated when registering the Gitlab-Runner with the Gitlab cloud.
If if changing the config.toml via the ConfigMap does not work, how should I then change the configuration? I cannot find anything about this in Gitlab or Gitlab documentation.
Inside the mapping you can try to append the volume and the extra configuration parameters:
# Add docker volumes
cat >> /home/gitlab-runner/.gitlab-runner/config.toml << EOF
[[runners.kubernetes.volumes.host_path]]
name = "var-run-docker-sock"
mount_path = "/var/run/docker.sock"
EOF
I did the runner deployment using a helm chart; I guess you did the same, in the following link you will find more information about the approach I mention: https://gitlab.com/gitlab-org/gitlab-runner/issues/2578
If after appending the config your pod is not able to start, check the logs, I did test the appending approach and had some errors like "Directory not Found," and it was because I was appending in the wrong path, but after fixing those issues, the runner works fine.
Seems to me you should be modifying config.template.toml (within your relevant configmap, that is)
If you want modify existing config.toml in /home/gitlab-runner/.gitlab-runner you need to set environment variables in deployment. For example, this is default set of variables in case you have installed gitlab-runner by pressing install button in gitlab.
Environment:
CI_SERVER_URL: http://git.example.com/
CLONE_URL:
RUNNER_REQUEST_CONCURRENCY: 1
RUNNER_EXECUTOR: kubernetes
REGISTER_LOCKED: true
RUNNER_TAG_LIST:
RUNNER_OUTPUT_LIMIT: 4096
KUBERNETES_IMAGE: ubuntu:16.04
KUBERNETES_PRIVILEGED: true
KUBERNETES_NAMESPACE: gitlab-managed-apps
KUBERNETES_POLL_TIMEOUT: 180
KUBERNETES_CPU_LIMIT:
KUBERNETES_CPU_LIMIT_OVERWRITE_MAX_ALLOWED:
KUBERNETES_MEMORY_LIMIT:
KUBERNETES_MEMORY_LIMIT_OVERWRITE_MAX_ALLOWED:
KUBERNETES_CPU_REQUEST:
KUBERNETES_CPU_REQUEST_OVERWRITE_MAX_ALLOWED:
KUBERNETES_MEMORY_REQUEST:
KUBERNETES_MEMORY_REQUEST_OVERWRITE_MAX_ALLOWED:
KUBERNETES_SERVICE_ACCOUNT:
KUBERNETES_SERVICE_CPU_LIMIT:
KUBERNETES_SERVICE_MEMORY_LIMIT:
KUBERNETES_SERVICE_CPU_REQUEST:
KUBERNETES_SERVICE_MEMORY_REQUEST:
KUBERNETES_HELPER_CPU_LIMIT:
KUBERNETES_HELPER_MEMORY_LIMIT:
KUBERNETES_HELPER_CPU_REQUEST:
KUBERNETES_HELPER_MEMORY_REQUEST:
KUBERNETES_HELPER_IMAGE:
Modify existing values or add new ones - it will appear in correct section of config.toml.
Does anyone successfully setup kubernetes executor/runner on gitlab for CI jobs? I set up mine but its stucking on executing my pipeline indefinitely.
I'm running a runner as a docker container on top of kubernetes cluster and connecting to my gitlab instance for handling my CI builds.
Any working config file would be appreciated.
My runner configuration looks like this:
[[runners]]
name = "kube-executor"
url = "https://gitlab.example.ltd/"
token = "some-token"
executor = "kubernetes"
[runners.cache]
[runners.kubernetes]
host = "https://my-kubernetes-api-address:443"
ca_file = "/etc/ssl/certs/ca.crt"
cert_file = "/etc/ssl/certs/server.crt"
key_file = "/etc/ssl/certs/server.key"
image = "docker:latest"
namespace = "gitlab"
namespace_overwrite_allowed = "ci-.*"
privileged = true
cpu_limit = "1"
memory_limit = "1Gi"
service_cpu_limit = "1"
service_memory_limit = "1Gi"
helper_cpu_limit = "500m"
helper_memory_limit = "100Mi"
poll_interval = 5
poll_timeout = 3600
[runners.kubernetes.volumes]
this throws this error: ERROR: Job failed (system failure): Post https://my-kubernetes-api-address:443/api/v1/namespaces/gitlab/secrets: x509: certificate signed by unknown authority
you are using https, so where are the certs, are they self signed certs? if yes you have to mention --tls-cert-file and --tls-private-key-file flags in your configmap.
Copied from https://stackoverflow.com/a/43362697/432115
I am installing a ceph-cluster with one monitor node and one osd.
I am following the document: http://docs.ceph.com/docs/v0.86/start/quick-ceph-deploy/
During the step 5: Add the initial monitor(s) and gather the keys (new in ceph-deploy v1.1.3),
I am getting the following exception:
**[ceph-mon1][ERROR ] admin_socket: exception getting command descriptions: [Errno 2] No such file or directory**
[ceph-mon1][WARNIN] monitor: mon.ceph-mon1, might not be running yet
[ceph-mon1][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-mon1.asok mon_status
**[ceph-mon1][ERROR ] admin_socket: exception getting command descriptions: [Errno 2] No such file or directory
[ceph-mon1][WARNIN] monitor ceph-mon1 does not exist in monmap**
Just for reference my **ceph.conf** is as follows:
*[global]
fsid = 351948ba-9716-4a04-802d-28b5510bfeb0
mon_initial_members = ceph-mon1,ceph-admin,ceph-osd1
mon_host = xxx.yyy.zzz.78,xxx.yyy.zzz.147,xxx.yyy.zzz.135
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
filestore_xattr_use_omap = true
osd_pool_default_size = 2
public_addr = xxx.yyy.zzz.0*
I tried to understand all the questions related to sane on ceph user mailing list but there is no precise solution I found for this problem.
Can anyone help me on this?
Thanks in advance.
I faced the same errors was able to resolve the issue by adding my other ceph node's hostname & IpAdrress and by adding "public_network ="
The sections which I tweaked in ceph.conf are:
mon_initial_members =
mon_host =
public_network =
cat /etc/ceph/ceph.conf
[global]
fsid = 33cb5c76-a685-469e-8cdd-fee7c98c3f4d
mon_initial_members = ceph1,ceph2
mon_host = 192.168.61.39,192.168.61.40
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
filestore_xattr_use_omap = true
public_network = 192.168.61.0/24
And the running the command:
$ ceph-deploy --overwrite-conf mon create <ceph-node>
I had a similar issue...
My problem was that the alias hostname in my /etc/hosts on my deployment server and my target server was with a different hostname ....
Always make sure your hostname on the server is the same in the ceph.conf and the correct IP- HOSTNAME are the same in /etc/hosts on your deployment box ...
StarCluster seems to use IPython 0.13.1 by default. Is there a way to upgrade this to IPython 2.3.1? Can it be done via the config file? Or manually after the cluster is started?
Here is my config, with only minor security changes:
[global]
DEFAULT_TEMPLATE=iptemplate
REFRESH_INTERVAL=5
[aws info]
aws_access_key_id = XXXXXXXXXXXXXXXXXXXXX
aws_secret_access_key = xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
aws_region_name = us-west-2
aws_region_host = ec2.us-west-2.amazonaws.com
[keypair starcluster]
key_location = starcluster.pem
[plugin ipcluster]
SETUP_CLASS = starcluster.plugins.ipcluster.IPCluster
ENABLE_NOTEBOOK = True
NOTEBOOK_PASSWD = XXXX
[plugin ipclusterstop]
SETUP_CLASS = starcluster.plugins.ipcluster.IPClusterStop
[plugin ipclusterrestart]
SETUP_CLASS = starcluster.plugins.ipcluster.IPClusterRestartEngines
[plugin pypackages]
setup_class = starcluster.plugins.pypkginstaller.PyPkgInstaller
packages = scikit-learn, psutil, pandas
# Base configuration for IPython.parallel cluster
[cluster iptemplate]
KEYNAME = starcluster
CLUSTER_SIZE = 1
CLUSTER_USER = ipuser
CLUSTER_SHELL = bash
#REGION = us-east-1
NODE_IMAGE_ID = ami-706afe40 # REGION and NODE_IMAGE_ID go in pair
NODE_INSTANCE_TYPE = c1.xlarge # 8 CPUs
DISABLE_QUEUE = True # We don't need SGE, faster cluster startup
PLUGINS = pypackages, ipcluster
You can do it by updating setup.py. Add "ipython==2.3.1" to install_requires and rerun the setup command. It will update ipython to the version specified.