Issue generating pre-signed URL
Closed this issue · 7 comments
I have authenticated using the standard:
Sys.setenv( AWS_ACCESS_KEY_ID = "abc", AWS_SECRET_ACCESS_KEY = "123", AWS_REGION = "us-east-1" )
This authentication has worked to list buckets and retrieve objects. However, when I try this code:
bucket <- 'BUCKET_NAME' s3_key <- 'OBJECT_KEY s3()$generate_presigned_url( client_method='get_object', params=list(Bucket=bucket, Key=s3_key), expires_in=3600 )
A pre-signed URL is generated, but I am met with a:
InvalidRequest The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256. VGDS2GCV99JAA9ED /Iq9V2rLJmvXGYLofe+1uMZz20O/u25GzyTcDQUWExntjB3FUoRrtSL81YMi/aGs9DEMTTwvOdM=
I have checked and I am able to generate a pre-signed URL using the AWS console.
Any thoughts as to the error?
Ah this make sense, currently generate_presigned_url is using sha1 algorithm. Which was developed from boto3. However I might of missed something that should change the hashing algorithm.
Are you able to get a working generate_presigned_url from boto3?
import boto3
s3 = boto3.client("s3")
bucket = "your-bucket"
key = "your-key"
s3.generate_presigned_url(
"get_object",
Params = {"Bucket": bucket, "Key": key}
)From further investigation boto3 allows you to change default signature_version.
By default boto3 will use sha1 and signature_version = "s3v4" will use sha256.
import boto3
from botocore.config import Config
s3_default = boto3.Session(profile_name = "paws").client("s3")
s3_custom = boto3.Session(profile_name = "paws").client(
"s3", config=Config(signature_version="s3v4")
)
bucket = "my-bucket"
s3_default.generate_presigned_url(
"list_objects", Params = {"Bucket": bucket}
)
#> https://my-bucket.s3.amazonaws.com/?encoding-type=url&AWSAccessKeyId=ABCDEFGHIJK123456789&Signature=BruHGTRQffmrsKtNkGR6bw62GTQ%3D&Expires=1689154977
s3_custom_v2.generate_presigned_url(
"list_objects", Params={"Bucket":bucket}
)
#> https://my-bucket.s3.amazonaws.com/?encoding-type=url&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ABCDEFGHIJK123456789%2F20230712%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230712T084559Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=ad6bc4af7955f38eff5956bf3a7b38a066d9ec283e53063e71f71865a718e682For demo purposes we can hack our current paws implementation to do the same thing.
library(paws.common)
s3 <- paws::s3()
bucket = "my-bucket"
s3$generate_presigned_url_v2 <- function (client_method, params = list(), expires_in = 3600,
http_method = NULL, signature_version="default") {
stopifnot("\'client_method\' must to be a character" = is.character(client_method),
"\'params\' must be a list of parameters for client_method" = is.list(params),
"\'expires\' must be numeric" = is.numeric(expires_in),
"\'expires_in\' must be greater than 0" = expires_in > 0L,
"\`http_method\` must be a character" = (is.character(http_method) || is.null(http_method)))
pkg_name <- "paws.storage"
pkg_api <- "s3"
.pkg_api <- paste0(".", pkg_api)
tryCatch({
operation_fun <- get(sprintf("%s_%s", pkg_api, client_method),
envir = getNamespace(pkg_name))
}, error = function(err) {
stop(sprintf("Client does not have method: %s", client_method),
call. = FALSE)
})
operation_body <- body(operation_fun)
op <- eval(operation_body[[2]][[3]], envir = getNamespace("paws.common"))
original_params <- formals(operation_fun)
original_params <- if (!is.null(original_params))
original_params
else list()
if (!is.null(original_params))
original_params
else list()
param_check <- setdiff(names(params), names(original_params))
if (!identical(param_check, character(0)) || is.null(param_check)) {
stop(sprintf("Invalid parameter(s) [`%s`] for client method %s",
paste(param_check, collapse = "`, `"), client_method),
call. = FALSE)
}
kwargs <- as.list(modifyList(original_params, params))
input <- do.call(get(.pkg_api, envir = getNamespace(pkg_name))[[sprintf("%s_input",
client_method)]], kwargs)
output <- get(.pkg_api, envir = getNamespace(pkg_name))[[sprintf("%s_input",
client_method)]]()
config <- get_config()
svc <- get(.pkg_api, envir = getNamespace(pkg_name))[["service"]](config)
request <- new_request(svc, op, input, output)
request$expire_time <- expires_in
request <- do.call("build", list(request = request), envir = getNamespace("paws.common"))
signer <- ifelse(signature_version=="s3v4", "s3v4_sign_request_handler", "sign_v1_auth_query")
request <- do.call(signer, list(request = request),
envir = getNamespace("paws.common"))
if (!is.null(http_method)) {
request$http_request$url$scheme <- http_method
}
return(do.call("build_url", list(url = request$http_request$url),
envir = getNamespace("paws.common")))
}
s3$generate_presigned_url_v2(
'list_objects',
params=list(Bucket=bucket)
)
#> [1] "https://my-bucket.s3.eu-west-1.amazonaws.com/?AWSAccessKeyId= ABCDEFGHIJK123456789&Expires=1689155836&Signature=0Yz7Jtg2dCL%2F2MCSTRCdYtcc6IV9XkMbFQakAXk0h2g%3D"
s3$generate_presigned_url_v2(
'list_objects',
params=list(Bucket=bucket),
signature_version = "s3v4"
)
#> [1] "https://my-bucket.s3.eu-west-1.amazonaws.com/?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential= ABCDEFGHIJK123456789%2F20230712%2Feu-west-1%2Fs3%2Faws4_request&X-Amz-Date=20230712T085717Z&X-Amz-Expires=3600&X-Amz-Signature=ff3763f382d749cfe87dcc56e0753dbbf4b42531881d47b8d9d71db5a1076ddf&X-Amz-SignedHeaders=host"Created on 2023-07-12 with reprex v2.0.2
I believe I can have this in the next paws regen.
Hi @owbezick ,
paws 0.4.0 will opt to setting the signature_version in the config of the client. This is align with boto3 implementation.
# paws 0.4.0
library(paws.common)
s3 <- paws::s3(config(signature_version = "s3v4"))
s3$generate_presigned_url("list_objects", params = list(Bucket = "my-bucket"))import boto3
from botocore.config import Config
s3 = boto3.client("s3", config = Config(signature_version = "s3v4"))
s3.generate_presigned_url("list_objects", Params = {"Bucket" : "my-bucket"})I will update ticket paws 0.4.0 ready on r-universe and finally cran
paws v-0.4.0 has now been released to the cran. I will close this ticket for now.