How to Create Migration
dashawk opened this issue · 12 comments
dashawk commented
How can we create migrations using this package?
baopham commented
Are you referring to a create-table migration? just create a normal migration file:
<?php
use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class CreateFlightsTable extends Migration
{
public function up()
{
// follow dynamodb PHP sdk docs http://docs.aws.amazon.com/aws-sdk-php/v3/api/api-dynamodb-2012-08-10.html#createtable
}
public function down()
{
// follow http://docs.aws.amazon.com/aws-sdk-php/v3/api/api-dynamodb-2012-08-10.html#deletetable
}
}zoul0813 commented
DynamoDb does not require a schema, so the traditional migration pattern does not apply. You can use CloudFormation templates, and/or the AWS CLI, to create your DynamoDb tables programmatically.
I wrote a small artisan command that uses the AWS SDK and allows me to run a handful of commands against DynamoDB. One of these commands is "create-table" which takes the name(s) of a model and then loads up the JSON definition of the table from my database folder and creates the table, it also uses the models "version" property (another additive) to make sure that I have version controlled tables. I can share this command later today when I have access to my workstation.
zoul0813 commented
This is just a rough console command I wrote, it has a few conveniences built into it. I didn't write it with the intent to share it, so it's pretty rough around the edges.
<?php
## app/Console/Commands/DynamoDB.php
namespace App\Console\Commands;
use Illuminate\Support\Facades\App;
use Illuminate\Console\Command;
use Aws\DynamoDb\Exception\DynamoDbException;
use Symfony\Component\Process\Process;
use Symfony\Component\Process\Exception\ProcessFailedException;
class DynamoDB extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'dynamodb:exec {action} {--table=} {--id=} {--user=} {--tag=true}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'DynamoDB Wrapper';
private $client;
private $config = [];
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
if(env('DYNAMODB_LOCAL')) {
$this->config['endpoint'] = env('DYNAMODB_LOCAL_ENDPOINT');
}
$this->client = App::make('aws')->createClient('dynamodb', $this->config);
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$action = $this->argument('action');
$tables = $this->option('table');
if(!empty($tables)) {
$names = explode(',', $tables);
$arr = [];
foreach($names as $name) {
$modelClass = 'App\\' . $name;
$model = new $modelClass();
$arr[$name] = $model->getTable();
}
$tables = $arr;
}
$ids = $this->option('id');
if(!empty($ids)) {
$ids = explode(',', $ids);
}
$users = $this->option('user');
if(!empty($ids)) {
$users = explode(',', $users);
}
$tag = $this->option('tag');
if($tag == 'false') $tag = false;
else $tag = true;
echo "Tagging: " . ($tag ? 'Enabled' : 'Disabled') . "\n";
switch($action) {
case 'list-tables': {
$this->info('Listing Tables');
$tables = $this->client->listTables();
foreach($tables['TableNames'] as $name=>$table) {
$this->line($table);
}
} break;
case 'scan': {
foreach($tables as $table) {
echo "Scanning: " . $table . "\n";;
$items = $this->client->getIterator('Scan', [
'TableName' => $table,
]);
foreach($items as $item) {
$this->line(json_encode($item));
}
}
} break;
case 'count': {
if(empty($tables)) $tables = $this->getAllTables();
foreach($tables as $table) {
echo $table;
$output = $this->client->describeTable([
'TableName' => $table,
]);
echo " Items: " . $output['Table']['ItemCount'] . "\n";
}
} break;
case 'describe-table': {
foreach($tables as $table) {
echo $table . ": \n";
$output = $this->client->describeTable([
'TableName'=>$table
]);
}
} break;
case 'delete': {
if(count($tables) !== 1) { $this->error('You must provide a single table to delete item(s) from.'); return; }
if(count($users) > 1 && count($users) != count($ids)) { $this->error('If passing User IDs, the number of Users must match the number of IDs'); return ; }
$table = $tables[0];
if($this->confirm('Are you sure you want to delete "' . implode(',', $ids) . '" from "' . $table . '"')) {
foreach($ids as $index=>$id) {
try {
$key = [
'id' => ['S' => $id],
];
if(count($users) == 1) {
$key['user_id'] = ['S' => $users[0]];
}
if(count($users) == count($ids)) {
$key['user_id'] = ['S' => $users[$index]];
}
$response = $this->client->deleteItem([
'TableName' => $table,
'Key' => $key,
]);
$meta = $response->get('@metadata');
if($meta['statusCode'] == 200) {
$this->line('Successfully deleted ' . $id . ' from ' . $table);
} else {
$this->error('Failed to delete ' . $id . ' from ' . $table);
}
} catch(DynamoDbException $e) {
$this->error('Unable to delete ' . $id . ' from ' . $table);
$this->line($e->getMessage());
}
}
}
} break;
case 'get': {
if(count($tables) !== 1) { $this->error('You must provide a single table to delete item(s) from.'); return; }
if(count($users) > 1 && count($users) != count($ids)) { $this->error('If passing User IDs, the number of Users must match the number of IDs'); return ; }
$table = $tables[0];
foreach($ids as $index=>$id) {
try {
$key = [
'id' => ['S' => $id],
];
if(count($users) == 1) {
$key['user_id'] = ['S' => $users[0]];
}
if(count($users) == count($ids)) {
$key['user_id'] = ['S' => $users[$index]];
}
$response = $this->client->getItem([
'ConsistentRead' => true,
'TableName' => $table,
'Key' => $key,
]);
if(empty($response['Item'])) $this->error('Item does not exists, ' . $id . ' in ' . $table);
else var_dump($response['Item']);
} catch(DynamoDbException $e) {
$this->error('Unable to delete ' . $id . ' from ' . $table);
$this->line($e->getMessage());
}
}
} break;
case 'delete-table': {
foreach($tables as $table) {
if($this->confirm('Are you sure you want to delete "' . $table . '"')) {
try {
$this->client->deleteTable([
'TableName' => $table
]);
$this->line($table . ' deleted.');
} catch(DynamoDbException $e) {
$this->error('Unable to delete ' . $table);
$this->line($e->getMessage());
}
}
}
} break;
case 'create-table': {
foreach($tables as $basic=>$table) {
$path = join(DIRECTORY_SEPARATOR, [base_path(), 'database', 'schemas', $basic . '.json']);
$this->line('Loading: ' . $path);
$input = json_decode(file_get_contents($path), true);
$env = config('app.env');
if($env !== 'local') {
$input['TableName'] = $table;
}
$error = json_last_error();
if($error) {
$this->line('ERROR: ' . $error);
}
$table = $this->client->createTable($input);
if($tag) {
$description = $table->get('TableDescription');
echo "ARN: "; var_dump($description['TableArn']);
if(!empty($description['TableArn'])) {
$tags = config('aws.tags', null);
if($tags !== null) {
echo "Waiting for 5s before calling TagResource ... ";
sleep(5); // wait, because it takes a moment for the table to become available to tag
echo "Done.\n";
$errors = 0;
while($errors < 3) {
try {
$tagged = $this->client->tagResource([
'ResourceArn' => $description['TableArn'],
"Tags" => $tags
]);
break;
} catch(Exception $e) {
$this->line($e->getMessage());
$errors++;
}
}
echo "Tagged: "; var_dump($tagged);
}
}
}
$this->line('Created ' . $table);
}
} break;
default: {
$this->error('Unsupported DynamoDB command');
}
}
}
private function getAllTables() {
$output = [];
$tables = $this->client->listTables();
foreach($tables['TableNames'] as $name=>$table) {
$output[] = $table;
}
return $output;
}
}
And in my database/schemas folder, I have the JSON files ... which look like this
{
"AttributeDefinitions": [
{
"AttributeName": "id",
"AttributeType": "S"
}
],
"TableName": "TableName",
"KeySchema": [
{
"AttributeName": "id",
"KeyType": "HASH"
}
],
"ProvisionedThroughput": {
"ReadCapacityUnits": 1,
"WriteCapacityUnits": 1
},
"StreamSpecification": {
"StreamEnabled": true,
"StreamViewType": "NEW_AND_OLD_IMAGES"
},
"Tags": [
{ "Key": "ENVIRONMENT", "Value": "dev" }
]
}
The model looks something like this
<?php
namespace App;
class BaseModel extends DynamoDbModel
{
public $version = '0.0.0';
public function getTable()
{
$prefix = config('app.prefix');
$ver = $this->version;
return strtolower(implode('_', [$prefix, $this->table, $ver]));
}
}
dashawk commented
I agree with @zoul0813 , that is why I get confused on how to create the migration using php artisan make:migration command.
how about adding an example migration in the README.md so that I can have an idea on what to put inside the up() and down() closures.
zoul0813 commented
@dashawk I'm not sure what part of my comments you're agreeing with. I agree with @baopham, if you want to use the migration logic in Laravel. I proposed an alternative solution by providing a custom artisan command I wrote for managing my DynamoDB resources.
Here's a sample migration class, this uses a copy of the code from my 'create-table' block above, with a local PHP Array representing the table schema. It also includes a 'deleteTable' in the down() call.
Refer to the DynamoDB API Documentation for more options.
http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_CreateTable.html
If the schema has a "Tags" key in the array, then the code will wait 5s to give AWS enough time to create the resource before calling the tagResource method. If this fails, it will try again up to 3 times waiting 5s in between each call. You can store global/general tags for your project in the config/aws.php file under the 'tags' key, in a ['Key'=>'key', 'Value'=>'value'] format ...
Sample config/aws.php
<?php
use Aws\Laravel\AwsServiceProvider;
return [
'region' => env('AWS_REGION', 'us-east-1'),
'version' => 'latest',
'debug' => false,
'ua_append' => [
'L5MOD/' . AwsServiceProvider::VERSION,
],
'tags' => [
[ 'Key' => 'standardKey', 'Value' => 'standardValue' ],
[ 'Key' => 'secondKey', 'Value' => 'secondValue' ],
]
];
Sample Migration Class
<?php
use Illuminate\Database\Migrations\Migration;
class SampleTable extends Migration
{
private $client;
private $config = [];
public function __construct()
{
if(env('DYNAMODB_LOCAL')) {
$this->config['endpoint'] = env('DYNAMODB_LOCAL_ENDPOINT');
}
$this->client = App::make('aws')->createClient('dynamodb', $this->config);
}
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
$schema = [
"AttributeDefinitions" => [
[
"AttributeName" => "id",
"AttributeType" => "S"
]
],
"TableName" => "SampleTable",
"KeySchema" => [
[
"AttributeName" => "id",
"KeyType" => "HASH"
]
],
"ProvisionedThroughput" => [
"ReadCapacityUnits" => 1,
"WriteCapacityUnits" => 1
],
"StreamSpecification" => [
"StreamEnabled" => true,
"StreamViewType" => "NEW_AND_OLD_IMAGES"
],
"Tags" => [
[ "Key" => "AWSTagKey", "Value" => "SomeCustomValue" ],
]
];
$table = $this->client->createTable($schema);
if(!empty($schema['Tags'])) {
sleep(5); sleep(5); // wait 5s, table may not have been created yet
$description = $table->get('TableDescription');
if(!empty($description['TableArn'])) {
$tags = array_merge([], config('aws.tags', null), $schema['Tags']);
if($tags !== null) {
$errors = 0;
while($errors < 3) {
try {
$tagged = $this->client->tagResource([
'ResourceArn' => $description['TableArn'],
"Tags" => $tags
]);
break;
} catch(Exception $e) {
echo "EXCEPTION: " . $e->getMessage();
$errors++;
sleep(5); // wait 5s, table may not have been created yet
}
}
}
}
}
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
$this->client->deleteTable([
"TableName" => "SampleTable",
]);
}
}
@baopham feel free to clean this sample up and provide a reference to it in the README.md
baopham commented
Thanks @zoul0813! There are many good points in this discussion so I'll just reference it in the FAQ :)
bingalls commented
This is not ready for the README.md but could go into the wiki. First consider installing the following:
- aws/aws-sdk-php-laravel
- webpatser/laravel-uuid
There is still minor code to clean up. Still, I am running into issues. As of this writing, the latest version of Laravel is 5.7. It wants to write to the migrations mysql table, even after commenting out any existing migrations bundled with Laravel.
gonzalom commented
@JuniYadi , the error you have is because is trying to create a migration table in your default database, which is a MySQL database.
Migrations require a database to register which migrations have been run already, and which ones need to be run. So you need to configure one or use another method to create your table.
JuniYadi commented
allanzi commented
I gonna put this here as just an example of how I'm using it.
<?php
use BaoPham\DynamoDb\DynamoDbClientInterface;
use Illuminate\Database\Migrations\Migration;
use Illuminate\Support\Facades\App;
return new class extends Migration
{
protected DynamoDbClientInterface $dynamoDbClient;
public function __construct()
{
$this->dynamoDbClient = App::make(DynamoDbClientInterface::class);
}
/**
* Run the migrations.
*/
public function up(): void
{
$this->dynamoDbClient->getClient()->createTable([
'TableName' => 'user',
'AttributeDefinitions' => [
[
'AttributeName' => 'id',
'AttributeType' => 'S'
]
],
'KeySchema' => [
[
'AttributeName' => 'id',
'KeyType' => 'HASH'
]
],
'ProvisionedThroughput' => [
'ReadCapacityUnits' => 10,
'WriteCapacityUnits' => 20
]
]);
$this->dynamoDbClient->getClient()->waitUntil('TableExists', [
'TableName' => 'user'
]);
}
/**
* Reverse the migrations.
*/
public function down(): void
{
$this->dynamoDbClient->getClient()->deleteTable([
'TableName' => 'user'
]);
$this->dynamoDbClient->getClient()->waitUntil('TableNotExists', [
'TableName' => 'user'
]);
}
};BehroozBvk commented
Thank you for your good idea @allanzi . I also created a command to create a table using the code below to make the table easier, maybe it will help others.
Please create DynamoDBMakeCommand.php file:
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Filesystem\Filesystem;
class DynamoDBMakeCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'dynamodb:make {name}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Make an Table Class';
/**
* Filesystem instance
* @var Filesystem
*/
protected $files;
/**
* Create a new command instance.
* @param Filesystem $files
*/
public function __construct(Filesystem $files)
{
parent::__construct();
$this->files = $files;
}
/**
* Execute the console command.
*/
public function handle(): void
{
$path = $this->getSourceFilePath();
$this->makeDirectory(dirname($path));
$contents = $this->getSourceFile();
if (!$this->files->exists($path)) {
$this->files->put($path, $contents);
$this->components->info(sprintf('Migration [%s] created successfully.', $path));
} else {
$this->components->warn(sprintf('Migration [%s] already exits', $path));
}
}
/**
* Return the stub file path
* @return string
*
*/
public function getStubPath(): string
{
return base_path('stubs/dynamodb.stub');
}
/**
**
* Map the stub variables present in stub to its value
*
* @return array
*
*/
public function getStubVariables(): array
{
return [
'table' => $this->argument('name'),
];
}
/**
* Get the stub path and the stub variables
*
* @return bool|mixed|string
*
*/
public function getSourceFile(): mixed
{
return $this->getStubContents($this->getStubPath(), $this->getStubVariables());
}
/**
* Replace the stub variables(key) with the desire value
*
* @param $stub
* @param array $stubVariables
* @return string|array|bool
*/
public function getStubContents($stub, array $stubVariables = []): string|array|bool
{
$contents = file_get_contents($stub);
foreach ($stubVariables as $search => $replace) {
$contents = str_replace(['{{ table }}', '{{table}}'], $replace, $contents);
}
return $contents;
}
/**
* Get the full path of generate class
*
* @return string
*/
public function getSourceFilePath(): string
{
$prefixDate=date('Y_m_d_His');
$fileName = "{$prefixDate}_create_{$this->argument('name')}_table.php";
return database_path('migrations') . '/' . $fileName;
}
/**
* Build the directory for the class if necessary.
*
* @param string $path
* @return string
*/
protected function makeDirectory(string $path): string
{
if (!$this->files->isDirectory($path)) {
$this->files->makeDirectory($path, 0777, true, true);
}
return $path;
}
}also create folder stubs in root project and put on dynamodb.stub file:
<?php
use BaoPham\DynamoDb\DynamoDbClientInterface;
use Illuminate\Database\Migrations\Migration;
use Illuminate\Support\Facades\App;
return new class extends Migration
{
protected DynamoDbClientInterface $dynamoDbClient;
public function __construct()
{
$this->dynamoDbClient = App::make(DynamoDbClientInterface::class);
}
/**
* Run the migrations.
*/
public function up(): void
{
$this->dynamoDbClient->getClient()->createTable([
'TableName' => '{{ table }}',
'AttributeDefinitions' => [
[
'AttributeName' => 'id',
'AttributeType' => 'S'
]
],
'KeySchema' => [
[
'AttributeName' => 'id',
'KeyType' => 'HASH'
]
],
'ProvisionedThroughput' => [
'ReadCapacityUnits' => 10,
'WriteCapacityUnits' => 20
]
]);
$this->dynamoDbClient->getClient()->waitUntil('TableExists', [
'TableName' => '{{ table }}'
]);
}
/**
* Reverse the migrations.
*/
public function down(): void
{
$this->dynamoDbClient->getClient()->deleteTable([
'TableName' => '{{ table }}'
]);
$this->dynamoDbClient->getClient()->waitUntil('TableNotExists', [
'TableName' => '{{ table }}'
]);
}
};