We are using a PostgreSQL database with AWS RDS IAM authorization feature – which means that our application needs to refresh the authorization token every 10 minutes or so (since the token is valid for 15 minutes). This token is used as a database password and I need to periodically update it. We are using the Dropwizard framework which is taking advantage of Apache Commons DBCP Component that handles connection pooling.
I was able to enhance the configuration class so that it performs an AWS API call to get the token instead of reading the password from configuration file. However this works only once, during application startup, for 15 minutes. I would like to call AWS API for the token perdiodically and handle the creation of connections as well as invalidating old ones.
import org.jooq.Configuration;
import org.jooq.impl.DefaultConfiguration;
import io.dropwizard.setup.Environment;
import org.example.myapp.ApplicationConfiguration;
// more less relevant imports...
#Override
public void run(ApplicationConfiguration configuration, Environment environment) {
Configuration postgresConfiguration = new DefaultConfiguration().set(configuration.getDbcp2Configuration()
.getDataSource())
.set(SQLDialect.POSTGRES_10)
.set(new Settings().withExecuteWithOptimisticLocking(true));
// this DSLContext object needs to be refreshed/recreated every 10 minutes with the new password!
KeysDAO.initialize(DSL.using(postgresConfiguration));
// rest of the app's config
}
How can I implement such a connection recreation mechanism? The org.jooq.ConnectionProvider looks promising, but I need some more guidance on how to inject the password on a periodic basis (and implement a custom ConnectionProvider). Any hints would be greatly appreciated.
EDIT:
This morning I was able to confirm that after a fresh deployment the database interaction is possible, and after exactly 15 minutes I'm getting first exceptions:
org.postgresql.util.PSQLException: FATAL: PAM authentication failed for user "jikg_service"
at org.postgresql.core.v3.ConnectionFactoryImpl.doAuthentication(ConnectionFactoryImpl.java:514)
at org.postgresql.core.v3.ConnectionFactoryImpl.tryConnect(ConnectionFactoryImpl.java:141)
at org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:192)
at org.postgresql.core.ConnectionFactory.openConnection(ConnectionFactory.java:49)
at org.postgresql.jdbc.PgConnection.<init>(PgConnection.java:195)
at org.postgresql.Driver.makeConnection(Driver.java:454)
at org.postgresql.Driver.connect(Driver.java:256)
at org.apache.commons.dbcp2.DriverConnectionFactory.createConnection(DriverConnectionFactory.java:39)
at org.apache.commons.dbcp2.PoolableConnectionFactory.makeObject(PoolableConnectionFactory.java:256)
at org.apache.commons.pool2.impl.GenericObjectPool.create(GenericObjectPool.java:868)
at org.apache.commons.pool2.impl.GenericObjectPool.ensureIdle(GenericObjectPool.java:927)
at org.apache.commons.pool2.impl.GenericObjectPool.ensureMinIdle(GenericObjectPool.java:906)
at org.apache.commons.pool2.impl.BaseGenericObjectPool$Evictor.run(BaseGenericObjectPool.java:1046)
at java.base/java.util.TimerThread.mainLoop(Timer.java:556)
at java.base/java.util.TimerThread.run(Timer.java:506)
Suppressed: org.postgresql.util.PSQLException: FATAL: pg_hba.conf rejects connection for host "172.30.19.218", user "my_db_user", database "my_db_development", SSL off
at org.postgresql.core.v3.ConnectionFactoryImpl.doAuthentication(ConnectionFactoryImpl.java:514)
at org.postgresql.core.v3.ConnectionFactoryImpl.tryConnect(ConnectionFactoryImpl.java:141)
at org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:201)
... 12 common frames omitted
Those exceptions are repeated every minute.
I owe you all an explanation on this one. I forgot to mention one significant detail - we're actually using a modified version of Dropwizard developed in-house that uses bundled Apache Commons DBCP (which afaik is not officially part of Dropwizard) as well as other components. I ended up dropping Apache Commons DBCP in favor of HikariCP - which made it possible to update the pool configuration at runtime. Although not officially supported, the creator of the library hinted that it might work, and in our scenario it indeed worked. Below is a sample solution.
import org.jooq.Configuration;
import org.jooq.impl.DefaultConfiguration;
import io.dropwizard.setup.Environment;
import org.example.myapp.ApplicationConfiguration;
// more less relevant imports...
#Override
public void run(ApplicationConfiguration configuration, Environment environment) {
HikariDataSource hikariDataSource = loadDatabaseConfiguration(configuration.getDatabaseConfiguration());
new DbConfigurationLoader(hikariDataSource).start();
// this DSLContext object now has the reference to DataSource object that has an always-fresh password!
KeysDAO.initialize(DSL.using(hikariDataSource, SQLDialect.POSTGRES_10, new Settings().withExecuteWithOptimisticLocking(true)));
// rest of the app's config
}
private HikariDataSource loadDatabaseConfiguration(DatabaseConfiguration configuration) {
HikariDataSource hikariDataSource = new HikariDataSource();
hikariDataSource.setJdbcUrl(configuration.getJdbcUrl());
hikariDataSource.setDriverClassName(configuration.getDriverClassName());
hikariDataSource.setMinimumIdle(configuration.getMinimumIdle());
hikariDataSource.setMaximumPoolSize(configuration.getMaximumPoolSize());
hikariDataSource.setUsername(configuration.getJdbcUser());
return hikariDataSource;
}
private class DbConfigurationLoader extends Thread {
private final HikariDataSource hikariDataSource;
private final RdsTokenProvider rdsTokenProvider;
public DbConfigurationLoader(HikariDataSource hikariDataSource) {
this.rdsTokenProvider = new RdsTokenProvider();
this.hikariDataSource = hikariDataSource;
}
#Override
public void run() {
while (true) {
hikariDataSource.setPassword(rdsTokenProvider.getToken());
try {
Thread.sleep(/* token is valid for 15 minutes, so it makes sense to refresh it more often */);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
}
Hope this saves somebody some time in the future.
Related
I'm posting the question here just to be sure I'm not barking on the wrong tree.
How to get the number of connections used (and free) to the MongoDB, but from a client perspective (eg. Java client), using the 4.x driver?
There are posts regarding using the serverStatus(Get the number of open connections in mongoDB using java), but it presumes having 'admin' access to the MongoDB. Using a 'regular user'(an db user with lower privileges (e.g access to only one database)) cannot run the serverStatus(). But this provides only a view from the server-side (there are N connections from IP x).
Other posts mentioned how to setup the connection pool size (eg. using the MongoClients.create(MongoClientSettings settings) (see the 4.x API reference (https://mongodb.github.io/mongo-java-driver/4.0/apidocs/mongodb-driver-sync/com/mongodb/client/MongoClients.html)):
MongoCredential credential = MongoCredential.createCredential(
username,
"admin",
password.toCharArray());
MongoClient mongoClient = MongoClients.create(MongoClientSettings.builder()
.applyToClusterSettings(
builder -> builder.hosts(Arrays.asList(new ServerAddress(hostname, portNumber))))
.credential(credential)
.applyToConnectionPoolSettings(builder -> builder
.minSize(connectionPoolMinimumSize)
.maxSize(connectionPoolMaximumSize))
.readConcern(readConcern)
.readPreference(readPreference)
.writeConcern(writeConcern)
.build());
But none provided means to get the used and available connections the connection pool.
As mentioned by Oleg, using the ConnectionPoolListener would be a way, but that is available only in the 3.x drivers. The ConnectionPoolListener methods are marked as deprecated on 4.x (although it is still mentioned in the JMX Monitoring section (http://mongodb.github.io/mongo-java-driver/4.0/driver-reactive/reference/monitoring/).
You can use connection pool monitoring which is described here to keep track of connection states, and deduce the counts you are looking for.
I don't know if Java driver exposes the counters you are looking for as public APIs; many drivers don't.
Finally got this working:
created a custom connection pool listener, implementing the com.mongodb.event.ConnectionPoolListener...
public class CustomConnectionPoolListener implements ConnectionPoolListener {
...
}
... and having the stats counters updated on a store (accessible later)
#Override
public void connectionCreated(ConnectionCreatedEvent event) {
ConnectionPoolStatsPOJO cps = mongoConnectionPoolList.get(connectionPoolAlias);
cps.incrementConnectionsCreated();
mongoConnectionPoolList.put(connectionPoolAlias, cps);
}
attached this custom connection pool listener to the MongoClient connection:
ConnectionPoolListener customConnPoolListener = new CustomConnectionPoolListener(...); /* added some references in the */
...
MongoClientSettings mongoClientSettings = MongoClientSettings.builder()
.applicationName(applicationName)
.applyConnectionString(connURI)
.credential(credential)
.readConcern(readConcern)
.readPreference(readPreference)
.writeConcern(writeConcern)
.applyToConnectionPoolSettings(builder -> builder
.minSize(connectionPoolMinimumSize)
.maxSize(connectionPoolMaximumSize)
.addConnectionPoolListener(customConnPoolListener)
)
.retryWrites(true)
.retryReads(true)
.build();
...
MongoClient mongoClient = MongoClients.create(mongoClientSettings);
....
finally, to access the connection pool stats, just have to query out the store:
ConnectionPoolStatsPOJO connectionPoolStats = MongoDB_ConnectionPool_Repository.getInstance().getMongoConnectionPoolList().get(connectionPoolAlias);
Therefore, thanks to "#D. SM" for pointing to the right direction.
I'm using MVC5 with classic ADO.NET objects such as sqldatareader and sqldataadapter and sqlconnection and so on....
My controllers are creating a connection while initializing because I need to send the request object to the class holding the sqlconnection for something irrelevant to the question so my controller has an override void
protected override void Initialize(RequestContext requestContext)
{
base.Initialize(requestContext);
db = new db(Request);
db.Connect();
}
Where db is my class and the method (connect) will create the sqlconnection object and open a connection...
and to close the connection I used the controller's dispose method as follows
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
if (db != null)
{
db.Close();
db = null;
}
}
and everything works fine then at one moment I got a weird server error (can't connect to db) please notice that my host is smarterasp.net
I can connect to database remotely using my home computer and I can connect to the web host as well so the problem is between my webhost and my database host, or between my application and my database host...
or it could be something related to the connection pooling even though the server error doesn't give me any details or stack trace(hens error is not inside my app thread)....
and I've fixed the problem by opening (remote iis) tab of smarterasp.net's control panel and clicked on (fix ACL) which I have no idea what it does but it fixed my problem.... temporarily :( unfortunately the problem reoccurred many times after that
so my question is in short format
is it good practice to open the connection while I'm initializing the controller and close it while the controller disposing???
and what do you think the error reason is??
finally I want to apologize if the question wasn't clear enough because English is not my first language (obviously)....
thanks a lot
so my question is in short format is it good practice to open the
connection while I'm initializing the controller and close it while
the controller disposing???
I do not think that is a good approach. You shouldn't open / close database connections and / or access the database from your controller. The controller should be as "thin" as possible. Additionally - the connection should be kept open for as short a period of time as possible and let ADO.NET connection pooling handle the details for you.
I also recommend wrapping your connection in a using block as it will implicitly call the close method:
using (SqlConnection connection = new SqlConnection(
connectionString))
{
SqlCommand command = new SqlCommand(queryString, connection);
command.Connection.Open();
command.ExecuteNonQuery();
}
You did not say what the exact error is. At first glance, you aren't even checking to see if the connection is open before you try closing it. You should check the connection state before you try to explicitly close it and this should happen outside of the controller. Though I recommend that you wrap your SqlConnection in a using block (mentioned above).
EDIT
I read your comment. You are trying to manage the connection within the context of the controller's lifecycle and I suspect this is your issue.
If you were using Entity Framework (or possibly an another ORM), an IoC with "per-request lifestyle" - then the IoC container would properly dispose your context (connection) at the end of each request, and serve a new instance at each new one.
Perhaps this an option you can explore if you want to manage your database connection this way.
I'm using jboss keycloak 1.5 final version.
I developed my custom user federation provider interfacing with keycloak properties and my user enterprise database.
My need is to send up to user the login interface custom error messages based on particular specific error related to my legacy user db.
I saw keycloak themes have a resources folder by which i can localize and add new messages. Then i can reference them by angular js using
$myMessage
notation. The problem is i want to rise up a message from keycloak server. My user federation provider implements UserFederationProvider interface. So i should have to override:
#Override
public CredentialValidationOutput validCredentials(RealmModel realm, UserCredentialModel credential) {
LOGGER.info("validCredentials(realm, credential)");
return CredentialValidationOutput.failed();
}
which seems to be the method i was looking for just because CredentialValidationOutput contains custom messages to be sent as validation output. The problem is this method is never called.
Why?
I'll post the answer found on my own.
It's necessary to develop your own Authenticator. For example refer to Keycloak UsernameAndForm and UsernameAndFormFactory implementation.
You can find them on Keycloak github source code:
https://github.com/keycloak/keycloak/tree/master/services/src/main/java/org/keycloak/authentication/authenticators/browser
The main validation method are:
public boolean validateUserAndPassword(AuthenticationFlowContext context, MultivaluedMap<String, String> inputData) {
...
}
public boolean validatePassword(AuthenticationFlowContext context, UserModel user, MultivaluedMap<String, String> inputData) {
...
}
From your custom user federation provider you can throw your custom exception and catch them in the two methods above adding:
catch (YourCustomException ex){
...
Response challengeResponse = context.form()
.setError("YOUR ERROR MESSAGE", me.getMandator()).createLogin();
context.failureChallenge(AuthenticationFlowError.INVALID_USER, challengeResponse);
return false;
}
Of course in your project you have to add
META-INF/service/org.keycloak.authentication.AuthenticatorFactory
In which you specify the full qualified name of your AuthenticatorFactory.
For a valid guide make reference to Keycloak User Guide 1.6.1 Final. Chapter 33.3
I am learning how to cache objects using memcached with the spymemcached client from spymemcached examples
MemcachedClient c=new MemcachedClient(new InetSocketAddress("hostname", portNum));
// Store a value (async) for one hour
c.set("someKey", 3600, someObject);
// Retrieve a value (synchronously).
Object myObject=c.get("someKey");
I have noted that each time I want to cache or retrieve an object I create a new memcached client which am assuming is a new connection and that memcached has no connection pooling mechanism therefore users are advised to cache the connections to decrease overhead for reconnecting from this question opening closing and reusing connections.
My question is how do I cache this connection? Can someone please give me an example I can start from.
If you are wondering what I have tried, I tried to put my connection in the memcached but then I realized that I have to create a connection to get it. :)
Thanks in advance.
I have noted that each time I want to cache or retrieve an object I
create a new memcached client which am assuming is a new connection
Don't do this; spymemcache uses a single connection for ALL I/O to and from memcache; it's all done asychronously; from spymemcache docs...
Each MemcachedClient instance establishes and maintains a single
connection to each server in your cluster.
Just do the following once in your app; make sure the client is available to other services in your app so they can access it.
MemcachedClient memClient = new MemcachedClient(new InetSocketAddress(host, port));
Use memClient for I/O with memcache; no need to create a new instance of MemcachedClient each time; done. The link you provided answers all of your questions.
What is your deployment? web-app or standalone?
This just means that you should use reuse the connections that you open as opposed to opening a connection for each request. It doesn't make sense to store a connection instance in memcached.
Cacheing the connection in the case means caching it in your application (keeping it in memory and open), not actually storing the connection in memcached.
I did a little more research and stumbled on this question
Then I came up with my solution as,
first created a contextlistener
public class ContextListener implements ServletContextListener {
#Override
public void contextInitialized(ServletContextEvent sce) {
Memcached.createClient();
}
#Override
public void contextDestroyed(ServletContextEvent sce) {
}
}
then i added the listener to the deployment discriptor by adding these lines to web.xml
<listener>
<description>Used with memcached to initialize connection</description>
<listener-class>com.qualebs.managers.ContextListener</listener-class>
</listener>
I created a class Memcached and added these methods
static void createClient() {
try {
client = new MemcachedClient(new InetSocketAddress("localhost", 11211));
} catch (IOException ex) {
Logger.getLogger(Memcached.class.getName()).log(Level.SEVERE, ex.getMessage(), ex);
}
}
static MemcachedClient getClient() throws IOException {
return client;
}
Now anywhere I need to use memcached connection just call Memcached.getClient()
I hope that will help anybody else out there with the same question.
I want to use Shiro on my next web project but I do not know a good (if not the best) strategy to manage users ([users] in shiro.ini).
Is it best to create Shiro user for every registered member?
Or create a single Shiro user then for every member just store it to some database and acces it via that Shiro user?
If you would go for #1, how would you manage/automate it? Most of the projects I worked on opted for #2.
Thanks
Configuring users in shiro.ini is not a good option for production environment. It can be used only if you have a small number of user accounts and you don't need to create or change accounts at runtime. It is mostly used for testing.
It is better for almost all projects to use some storage to keep all user accounts. It can be database or some external authentication engine, like ldap, cas or even oauth.
You can just use Stormpath as your user/group store. Drop in the Shiro integration and boom - instant user/group data store for Shiro-enabled applications with a full management UI and Java SDK.
It even helps automate things like 'forgot password' emails and account email verification. It's free for many usages too. You can see the Shiro sample app using Stormpath as an example.
Shiro provides multiple ways to configure users. Take a look at the possible Realm configurations here.
If none of these satisfy your needs, you could even write a custom Realm for your application, that can, say, pull user info from a NoSQL database, or get info from a SAML response, or use OAuth2. It's definitely not advisable to create any user details in shiro.ini in production. To give a notion of what custom realms might look like, here's an example where I created a SAML2 based user authc and authz: shiro-saml2.
PLease do not use only one user for everyone. Avoid this option.
Much better to use one user(account) per user.
In shiro, you can have the RDMS Realm that allows you to use a simple database like mysql in order to store your user /account / permissions. :)
Clone this project, (that is not mine), and get started in 1 minute! :)
shiro/mysql GIT example
Enjoy it :)
Shiro provide implementing your own realm as per your requirement.
Create a simple realm in which you can manage details, login, permissions and roles.
You can use jdbc, Hibernate, or any other authentication manner to manage them.
Configure this realm to your ini or whatever way you using in your project.
Now Shiro will automatically invoke methods of your realm class to look for credential, permissions, roles.
For ex I have a shiro hibernate realm I used my hibernate code to manage users in my db.
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import org.apache.shiro.authc.AuthenticationException;
import org.apache.shiro.authc.AuthenticationInfo;
import org.apache.shiro.authc.AuthenticationToken;
import org.apache.shiro.authc.SimpleAuthenticationInfo;
import org.apache.shiro.authc.UsernamePasswordToken;
import org.apache.shiro.authc.credential.CredentialsMatcher;
import org.apache.shiro.authz.AuthorizationInfo;
import org.apache.shiro.authz.SimpleAuthorizationInfo;
import org.apache.shiro.realm.AuthorizingRealm;
import org.apache.shiro.subject.PrincipalCollection;
/**
* #author Ankit
*
*/
public class PortalHibernateRealm extends AuthorizingRealm {
private static final Logger LOGGER = new Logger(
PortalHibernateRealm.class.toString());
/**
*
*/
public PortalHibernateRealm() {
super();
/*
* Set credential matcher on object creation
*/
setCredentialsMatcher(new CredentialsMatcher() {
#Override
public boolean doCredentialsMatch(AuthenticationToken arg0,
AuthenticationInfo arg1) {
UsernamePasswordToken token = (UsernamePasswordToken) arg0;
String username = token.getUsername();
String password = new String(token.getPassword());
/*
Check for credential and return true if found valid else false
*/
return false;
}
});
}
#Override
protected AuthorizationInfo doGetAuthorizationInfo(
PrincipalCollection principalCollection) {
Collection<String> permissionSet;
SimpleAuthorizationInfo info = null;
Long userId = (Long) principalCollection.getPrimaryPrincipal();
//Using thi principle create SimpleAuthorizationInfo and provide permissions and roles
info = new SimpleAuthorizationInfo();
return info;
}
#Override
protected AuthenticationInfo doGetAuthenticationInfo(
AuthenticationToken authcToken) throws AuthenticationException {
UsernamePasswordToken token = (UsernamePasswordToken) authcToken;
/*using this token create a SimpleAuthenticationInfo like
User user = UserUtil.findByEmail(token.getUsername());
*/
SimpleAuthenticationInfo authenticationInfo = new SimpleAuthenticationInfo(
primaryPrin, Password, screenName);
return authenticationInfo;
}
}