put-object with sigv4, error occurs: com.amazonaws.SdkClientException: Unable to execute HTTP request: Read after end of file
kisscelia opened this issue · 5 comments
kisscelia commented
Upcoming End-of-Support
- I acknowledge the upcoming end-of-support for AWS SDK for Java v1 was announced, and migration to AWS SDK for Java v2 is recommended.
Describe the bug
client configuration:
private ClientConfiguration getDefaultClientConfiguration() {
ClientConfiguration defaultClientConfiguration = new ClientConfiguration();
// Set connection timeout for initially establishing a connection.
defaultClientConfiguration.setConnectionTimeout(timeout);
// Set socket timeout for data to be transferred.
defaultClientConfiguration.setSocketTimeout(timeout);
// Set max connections.
defaultClientConfiguration.setMaxConnections(maxConnections);
// use expect continue HTTP/1.1 header.
defaultClientConfiguration.withUseExpectContinue(false);
if ((!proxyHost.equals("")) && (!proxyPort.equals(""))) {
defaultClientConfiguration.setProxyHost(proxyHost);
defaultClientConfiguration.setProxyPort(parms.getInt(PROXY_PORT_KEY));
}
return defaultClientConfiguration;
}
private AmazonS3 initClient() {
logger.debug("initialize S3 client with storage config: {}", parms);
ClientConfiguration clientConf;
if (sigv2) {
clientConf = getDefaultClientConfiguration().withSignerOverride("S3SignerType");
} else {
clientConf = getDefaultClientConfiguration();
}
AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey);
EndpointConfiguration myEndpoint = new EndpointConfiguration(endpoint, "");
client = AmazonS3ClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(myCredentials))
.withClientConfiguration(clientConf)
.withEndpointConfiguration(myEndpoint)
.withPathStyleAccessEnabled(pathStyleAccess)
.build();
logger.debug("S3 client has been initialized");
return client;
}
put-object method:
@Override
public void createObject(String container, String object, InputStream data, long length, Config config) {
super.createObject(container, object, data, length, config);
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(length);
metadata.setContentType("application/octet-stream");
metadata.setHeader("x-amz-storage-class", storageClass);
// 2021.7.27, sine. another way to put object, and set Read limit to 5GiB+1.
// https://github.com/awsdocs/aws-java-developer-guide/blob/master/doc_source/best-practices.rst
PutObjectRequest request = new PutObjectRequest(container, object, data, metadata);
request.getRequestClientOptions().setReadLimit((int)length + 1); // set limit to object length+1
try {
client.putObject(request);
} catch (AmazonServiceException ase) {
throw new StorageException(ase);
} catch (SdkClientException sce) {
sce.printStackTrace();
throw new StorageTimeoutException(sce);
}
}
Expected Behavior
put-object successfully.
Current Behavior
com.amazonaws.SdkClientException: Unable to execute HTTP request: Read after end of file
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleRetryableException(AmazonHttpClient.java:1219)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1165)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:814)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:781)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:755)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:715)
at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:697)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:561)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:541)
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5558)
at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5505)
at com.amazonaws.services.s3.AmazonS3Client.access$300(AmazonS3Client.java:423)
at com.amazonaws.services.s3.AmazonS3Client$PutObjectStrategy.invokeServiceCall(AmazonS3Client.java:6639)
at com.amazonaws.services.s3.AmazonS3Client.uploadObject(AmazonS3Client.java:1892)
at com.amazonaws.services.s3.AmazonS3Client.putObject(AmazonS3Client.java:1852)
at com.intel.cosbench.api.sio.SIOStorage.createObject(SIOStorage.java:290)
at com.intel.cosbench.driver.operator.Writer.doWrite(Writer.java:97)
at com.intel.cosbench.driver.operator.Writer.operate(Writer.java:81)
at com.intel.cosbench.driver.operator.AbstractOperator.operate(AbstractOperator.java:75)
at com.intel.cosbench.driver.agent.WorkAgent.performOperation(WorkAgent.java:197)
at com.intel.cosbench.driver.agent.WorkAgent.doWork(WorkAgent.java:177)
at com.intel.cosbench.driver.agent.WorkAgent.execute(WorkAgent.java:134)
at com.intel.cosbench.driver.agent.AbstractAgent.call(AbstractAgent.java:44)
at com.intel.cosbench.driver.agent.AbstractAgent.call(AbstractAgent.java:1)
at java.util.concurrent.FutureTask.run(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source)
Caused by: java.io.IOException: Read after end of file
at org.apache.commons.io.input.NullInputStream.read(NullInputStream.java:221)
at org.apache.commons.io.input.ProxyInputStream.read(ProxyInputStream.java:75)
at org.apache.commons.io.input.CountingInputStream.read(CountingInputStream.java:74)
at com.intel.cosbench.driver.generator.XferCountingInputStream.read(XferCountingInputStream.java:56)
at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:90)
at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:90)
at com.amazonaws.util.LengthCheckInputStream.read(LengthCheckInputStream.java:107)
at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:90)
at com.amazonaws.services.s3.internal.MD5DigestCalculatingInputStream.read(MD5DigestCalculatingInputStream.java:128)
at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:90)
at com.amazonaws.event.ProgressInputStream.read(ProgressInputStream.java:180)
at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:90)
at com.amazonaws.auth.AwsChunkedEncodingInputStream.setUpNextChunk(AwsChunkedEncodingInputStream.java:306)
at com.amazonaws.auth.AwsChunkedEncodingInputStream.read(AwsChunkedEncodingInputStream.java:172)
at org.apache.http.entity.InputStreamEntity.writeTo(InputStreamEntity.java:140)
at com.amazonaws.http.RepeatableInputStreamRequestEntity.writeTo(RepeatableInputStreamRequestEntity.java:160)
at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156)
at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152)
at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238)
at com.amazonaws.http.protocol.SdkHttpRequestExecutor.doSendRequest(SdkHttpRequestExecutor.java:63)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at com.amazonaws.http.apache.client.impl.SdkHttpClient.execute(SdkHttpClient.java:72)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1346)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1157)
... 26 more
Reproduction Steps
set sigv2 parameter to true, it's ok
set sigv2 to false, error occurs.
Possible Solution
no
Additional Information/Context
No response
AWS Java SDK version used
1.12.745
JDK version used
1.8.0_301
Operating System and version
CentOS 7.9