DataLakeServiceClient throws Type definition error: [simple type, class com.azure.core.http.netty.NettyAsyncHttpClient] while using Java SDK - azure-java-sdk

Question

I am trying access ADLS using Java SDK.
following is the configuration that i am using to get DataLakeServiceClient.
import com.azure.core.http.HttpClient;
import com.azure.core.http.netty.NettyAsyncHttpClientBuilder;
import com.azure.identity.ClientSecretCredential;
import com.azure.identity.ClientSecretCredentialBuilder;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.azure.core.credential.TokenCredential;
import com.azure.storage.common.StorageSharedKeyCredential;
import com.azure.storage.file.datalake.DataLakeDirectoryClient;
import com.azure.storage.file.datalake.DataLakeFileClient;
import com.azure.storage.file.datalake.DataLakeFileSystemClient;
import com.azure.storage.file.datalake.DataLakeServiceClient;
import com.azure.storage.file.datalake.DataLakeServiceClientBuilder;
import com.azure.storage.file.datalake.models.ListPathsOptions;
import com.azure.storage.file.datalake.models.PathAccessControl;
import com.azure.storage.file.datalake.models.PathAccessControlEntry;
import com.azure.storage.file.datalake.models.PathItem;
import com.azure.storage.file.datalake.models.PathPermissions;
import com.azure.storage.file.datalake.models.RolePermissions;
#RestController
#RequestMapping("/api")
public class AdlsController {
#Autowired
private Environment env;
#GetMapping("/getDataLakeServiceClient")
public DataLakeServiceClient GetDataLakeServiceClient() {
String endpoint = "https://" + env.getProperty("azure.storage.account-name") + ".dfs.core.windows.net";
ClientSecretCredential clientSecretCredential = new ClientSecretCredentialBuilder()
.clientId(env.getProperty("spring.security.oauth2.client.registration.azure.client-id"))
.clientSecret(env.getProperty("spring.security.oauth2.client.registration.azure.client-secret"))
.tenantId(env.getProperty("azure.activedirectory.tenant-id"))
.build();
DataLakeServiceClientBuilder builder = new DataLakeServiceClientBuilder();
HttpClient client = new NettyAsyncHttpClientBuilder()
.wiretap(true)
.build();
return builder.httpClient(client).credential(clientSecretCredential).endpoint(endpoint).buildClient();
}
}
The pom.xml has following dependencies:
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-oauth2-client</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-active-directory-spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage-spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-file-datalake</artifactId>
<version>12.1.1</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.11.1</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.11.1</version>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-core</artifactId>
<version>1.6.0</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-devtools</artifactId>
<scope>runtime</scope>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-commons</artifactId>
<version>2.2.1.RELEASE</version>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-blob</artifactId>
<version>12.6.0</version>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-core-http-netty</artifactId>
<version>1.5.3</version>
<scope>compile</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-spring-boot-bom</artifactId>
<version>${azure.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
On calling the controller the request is failing and I am getting following error:
org.springframework.http.converter.HttpMessageConversionException: Type definition error: [simple type, class com.azure.core.http.netty.NettyAsyncHttpClient]; nested exception is com.fasterxml.jackson.databind.exc.InvalidDefinitionException: No serializer found for class com.azure.core.http.netty.NettyAsyncHttpClient and no properties discovered to create BeanSerializer (to avoid exception, disable SerializationFeature.FAIL_ON_EMPTY_BEANS) (through reference chain: com.azure.storage.file.datalake.DataLakeServiceClient["httpPipeline"]->com.azure.core.http.HttpPipeline["httpClient"])
↵ at org.springframework.http.converter.json.AbstractJackson2HttpMessageConverter.writeInternal(AbstractJackson2HttpMessageConverter.java:323)
↵ at org.springframework.http.converter.AbstractGenericHttpMessageConverter.write(AbstractGenericHttpMessageConverter.java:104)
↵ at org.springframework.web.servlet.mvc.method.annotation.AbstractMessageConverterMethodProcessor.writeWithMessageConverters(AbstractMessageConverterMethodProcessor.java:287)
↵ at org.springframework.web.servlet.mvc.method.annotation.RequestResponseBodyMethodProcessor.handleReturnValue(RequestResponseBodyMethodProcessor.java:181)
↵ at org.springframework.web.method.support.HandlerMethodReturnValueHandlerComposite.handleReturnValue(HandlerMethodReturnValueHandlerComposite.java:82)
↵ at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:123)
↵ at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:879)
↵ at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:793)
↵ at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
↵ at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1040)
↵ at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:943)
↵ at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
↵ at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:898)
↵ at javax.servlet.http.HttpServlet.service(HttpServlet.java:634)
↵ at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
↵ at javax.servlet.http.HttpServlet.service(HttpServlet.java:741)
↵ at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
↵ at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
↵ at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
↵ at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
↵ at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:320)
↵ at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:126)
↵ at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:90)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
↵ at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:118)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
↵ at org.springframework.security.web.session.SessionManagementFilter.doFilter(SessionManagementFilter.java:137)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
↵ at org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:111)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
↵ at org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:158)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
↵ at org.springframework.security.web.savedrequest.RequestCacheAwareFilter.doFilter(RequestCacheAwareFilter.java:63)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
↵ at org.springframework.security.web.authentication.ui.DefaultLogoutPageGeneratingFilter.doFilterInternal(DefaultLogoutPageGeneratingFilter.java:52)
↵ at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
↵ at org.springframework.security.web.authentication.ui.DefaultLoginPageGeneratingFilter.doFilter(DefaultLoginPageGeneratingFilter.java:216)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
↵ at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:200)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
↵ at org.springframework.security.oauth2.client.web.OAuth2AuthorizationRequestRedirectFilter.doFilterInternal(OAuth2AuthorizationRequestRedirectFilter.java:160)
↵ at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
↵ at org.springframework.security.web.authentication.logout.LogoutFilter.doFilter(LogoutFilter.java:116)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
↵ at org.springframework.web.filter.CorsFilter.doFilterInternal(CorsFilter.java:92)
↵ at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
↵ at org.springframework.security.web.header.HeaderWriterFilter.doHeadersAfter(HeaderWriterFilter.java:92)
↵ at org.springframework.security.web.header.HeaderWriterFilter.doFilterInternal(HeaderWriterFilter.java:77)
↵ at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
↵ at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:105)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
↵ at org.springframework.security.web.context.request.async.WebAsyncManagerIntegrationFilter.doFilterInternal(WebAsyncManagerIntegrationFilter.java:56)
↵ at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
↵ at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:334)
↵ at org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:215)
↵ at org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:178)
↵ at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:358)
↵ at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:271)
↵ at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
↵ at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
↵ at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
↵ at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
↵ at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
↵ at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
↵ at org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:93)
↵ at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
↵ at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
↵ at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
↵ at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
↵ at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
↵ at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
↵ at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
↵ at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:202)
↵ at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
↵ at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541)
↵ at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139)
↵ at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
↵ at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
↵ at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
↵ at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:373)
↵ at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
↵ at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
↵ at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1590)
↵ at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
↵ at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
↵ at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
↵ at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
↵ at java.base/java.lang.Thread.run(Thread.java:835)
↵Caused by: com.fasterxml.jackson.databind.exc.InvalidDefinitionException: No serializer found for class com.azure.core.http.netty.NettyAsyncHttpClient and no properties discovered to create BeanSerializer (to avoid exception, disable SerializationFeature.FAIL_ON_EMPTY_BEANS) (through reference chain: com.azure.storage.file.datalake.DataLakeServiceClient["httpPipeline"]->com.azure.core.http.HttpPipeline["httpClient"])
↵ at com.fasterxml.jackson.databind.exc.InvalidDefinitionException.from(InvalidDefinitionException.java:77)
↵ at com.fasterxml.jackson.databind.SerializerProvider.reportBadDefinition(SerializerProvider.java:1277)
↵ at com.fasterxml.jackson.databind.DatabindContext.reportBadDefinition(DatabindContext.java:400)
↵ at com.fasterxml.jackson.databind.ser.impl.UnknownSerializer.failForEmpty(UnknownSerializer.java:71)
↵ at com.fasterxml.jackson.databind.ser.impl.UnknownSerializer.serialize(UnknownSerializer.java:33)
↵ at com.fasterxml.jackson.databind.ser.BeanPropertyWriter.serializeAsField(BeanPropertyWriter.java:728)
↵ at com.fasterxml.jackson.databind.ser.std.BeanSerializerBase.serializeFields(BeanSerializerBase.java:755)
↵ at com.fasterxml.jackson.databind.ser.BeanSerializer.serialize(BeanSerializer.java:178)
↵ at com.fasterxml.jackson.databind.ser.BeanPropertyWriter.serializeAsField(BeanPropertyWriter.java:728)
↵ at com.fasterxml.jackson.databind.ser.std.BeanSerializerBase.serializeFields(BeanSerializerBase.java:755)
↵ at com.fasterxml.jackson.databind.ser.BeanSerializer.serialize(BeanSerializer.java:178)
↵ at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider._serialize(DefaultSerializerProvider.java:480)
↵ at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.serializeValue(DefaultSerializerProvider.java:319)
↵ at com.fasterxml.jackson.databind.ObjectWriter$Prefetch.serialize(ObjectWriter.java:1513)
↵ at com.fasterxml.jackson.databind.ObjectWriter.writeValue(ObjectWriter.java:1005)
↵ at org.springframework.http.converter.json.AbstractJackson2HttpMessageConverter.writeInternal(AbstractJackson2HttpMessageConverter.java:317)
↵ ... 92 more
↵
In order to resolve this error i added following dependencies to to pom.xml but it does not seem to work
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-core-http-netty</artifactId>
<version>1.5.3</version>
<scope>compile</scope>
</dependency>
Can anyone help me out to solve this issue?

Answer 1

According to my test, we can use the following version sdk in spring boot application access azure data lake gen2
<properties>
<java.version>11</java.version>
<azure.version>2.2.4</azure.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-oauth2-client</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-active-directory-spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage-spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-file-datalake</artifactId>
<version>12.1.1</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-devtools</artifactId>
<scope>runtime</scope>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-spring-boot-bom</artifactId>
<version>${azure.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
Code
#RestController
public class HelloController {
#GetMapping("/")
public String getFile() throws IOException {
String clientId = "42e0d***********22d988c4";
String secret = "Gbx2eK64************Ijoae:";
String tenant = "e4c9ab4e-b*************0ba2a757fb";
String endpoint="https://testadls05.blob.core.windows.net/";
ClientSecretCredential clientSecretCredential = new ClientSecretCredentialBuilder()
.clientId(clientId)
.clientSecret(secret)
.tenantId(tenant)
.build();
DataLakeServiceClientBuilder builder = new DataLakeServiceClientBuilder();
HttpClient httpClient = new NettyAsyncHttpClientBuilder()
.wiretap(true)
.build();
DataLakeServiceClient client = builder.httpClient(httpClient).credential(clientSecretCredential).endpoint(endpoint).buildClient();
DataLakeFileSystemClient dataLakeFileSystemClient =client.getFileSystemClient("test");
DataLakeFileClient fileClient = dataLakeFileSystemClient.getFileClient("test.csv");
int dataSize = (int) fileClient.getProperties().getFileSize();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream(dataSize);
fileClient.read(outputStream);
String content =outputStream.toString();
outputStream.close();
return content;