Vapor 3.0.0 RC 1: Support Codable Models with more complex mapping
Closed this issue · 11 comments
I am getting nested JSON, which I am decoding / encoding in the model using init(from decoder: Decoder) and encode(to encoder: Encoder). While this compiles properly, the underlying table that is constructed does not contain all of the model's properties (e.g. the ones that are populated by those init and encode methods; also see this post about flattening JSON):
Note though, that this isn't Support for JSON!
I am on Vapor 3.0.0 RC 1 and Swift 4.1:
$ cat Package.swift
...
.package(url: "https://github.com/vapor/vapor.git", from:"3.0.0-rc"), //.branch("beta")),
.package(url: "https://github.com/vapor/fluent-mysql", from:"3.0.0-rc") // .branch("beta")),
...
$ swift --version
Apple Swift version 4.1 (swiftlang-902.0.41 clang-902.0.31)
Target: x86_64-apple-darwin17.4.0
$
Example JSON:
{
"type": 0,
"measurement": {
"t": 4.05,
"p": 77.81,
"h": 45.23
},
"published_at": "2018-02-22T11:39:15.159Z"
}Example Model:
enum MeasurementType: Int, Codable {
case placeA = 0
case placeB
case placeC
}
final class Measurement: Codable {
/// Model.ID
var id: Int?
var coreID: Core.ID?
var type: MeasurementType
var temperature: Double
var pressure: Double
var humidity: Double
var publishedAt: Date?
/// Timestampable.createdAt
var createdAt: Date?
/// Timestampable.updatedAt
var updatedAt: Date?
/// Initializer
init(id: ID? = nil, coreID: Core.ID?, type: MeasurementType, temperature: Double, pressure: Double, humidity: Double, publishedAt: Date) {
self.id = id
self.coreID = coreID
self.type = type
self.temperature = temperature
self.pressure = pressure
self.humidity = humidity
self.publishedAt = publishedAt
}
enum RootKeys: String, CodingKey {
case id
case coreID
case type
case measurement
case publishedAt = "published_at"
}
enum MeasurementKeys: String, CodingKey {
case temperature = "t"
case pressure = "p"
case humidity = "h"
}
init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: RootKeys.self)
id = try? container.decode(ID.self, forKey: .id)
coreID = try? container.decode(UUID.self, forKey: .coreID)
type = try! container.decode(MeasurementType.self, forKey: .type)
publishedAt = try container.decode(Date.self, forKey: .publishedAt)
let measurementContainer = try container.nestedContainer(keyedBy: MeasurementKeys.self, forKey: .measurement)
temperature = try measurementContainer.decode(Double.self, forKey: .temperature)
pressure = try measurementContainer.decode(Double.self, forKey: .pressure)
humidity = try measurementContainer.decode(Double.self, forKey: .humidity)
}
func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: RootKeys.self)
try? container.encode(id, forKey: .id)
try? container.encode(coreID, forKey: .coreID)
try? container.encode(type, forKey: .type)
try? container.encode(publishedAt, forKey: .publishedAt)
var measurementContainer = container.nestedContainer(keyedBy: MeasurementKeys.self, forKey: .measurement)
try measurementContainer.encode(temperature, forKey: .temperature)
try measurementContainer.encode(pressure, forKey: .pressure)
try measurementContainer.encode(humidity, forKey: .humidity)
}
}
/// MARK: Model
extension Measurement: Model {
/// See Model.Database
typealias Database = MySQLDatabase
/// See Model.ID
typealias ID = Int
/// See Model.idKey
static var idKey: WritableKeyPath<Measurement, Int?> {
return \Measurement.id
}
}
/// MARK: Relations
extension Measurement {
/// A relation to this pressure's core
var core: Parent<Measurement, Core>? {
return parent(\.coreID)
}
}
/// Stored date stamps
extension Measurement: Timestampable {
static var createdAtKey: WritableKeyPath<Measurement, Date?> {
return \.createdAt
}
static var updatedAtKey: WritableKeyPath<Measurement, Date?> {
return \.updatedAt
}
}
/// Support dynamic migrations.
extension Measurement: Migration { }
/// Support encoded to and decoded from HTTP messages.
extension Measurement: Content { }
/// Support using as a dynamic parameter in route definitions.
extension Measurement: Parameter { }@Joannis mentioned on Slack that this might be related to the MySQL schema? Right now I don't do anything schema related in Migration (create and delete do not seem to be available for MySQL 🤔) and let Vapor create the schema.
If I do try to add schema creation in the Model's Migration protocol conformance, it appears create and delete are not available:
code:
/// Support dynamic migrations.
extension Measurement: Migration {
/// See Migration.prepare
static func prepare(on connection: Database.Connection) -> Future<Void> {
connection.create(self) { builder in
try builder.field(for: \.id)
try builder.field(for: \.coreID)
try builder.field(for: \.type)
try builder.field(for: \.temperature)
try builder.field(for: \.pressure)
try builder.field(for: \.humidity)
try builder.field(for: \.publishedAt)
try builder.field(for: \.createdAt)
try builder.field(for: \.updatedAt)
}
}
/// See Migration.revert
static func revert(on connection: Database.Connection) -> Future<Void> {
return connection.delete(self)
}
}error:
Compile Swift Module 'App' (20 sources)
/.../Sources/App/Models/Measurement.swift:134:9: error: value of type 'MySQLDatabase.Connection' (aka 'MySQLConnection') has no member 'create'
connection.create(self) { builder in
^~~~~~~~~~ ~~~~~~
/.../Sources/App/Models/Measurement.swift:149:16: error: value of type 'MySQLDatabase.Connection' (aka 'MySQLConnection') has no member 'delete'
return connection.delete(self)
^~~~~~~~~~ ~~~~~~A couple of things here:
1: Fluent auto-migrations don't support nestedContainer w/ codable. Only the first level of properties have schema created for them, so this is working as expected.
2: Fluent MySQL needs a way to support nested structures. Right now this is not implemented. Fluent PostgreSQL supports this through the PostgreSQLType protocol. With this protocol, you can specify exactly how any custom type is serialized to the database.
When Fluent MySQL gets this feature, it will look something like this:
struct MeasurementData: MySQLJSONType {
var temperature: Double
var pressure: Double
var humidity: Double
}
final class Measurement: MySQLModel {
/// Model.ID
var id: Int?
var coreID: Core.ID?
var type: MeasurementType
var publishedAt: Date?
var measurement: MeasurementData
/// Timestampable.createdAt
var createdAt: Date?
/// Timestampable.updatedAt
var updatedAt: Date?
}Here MySQLJSONType would specify that you want this struct to be stored as nested JSON. You could also conform to MySQLType manually and choose exactly how this type gets parsed/serialized to MySQL (including the schema type: JSON, VARCHAR, BIGINT, etc)
3. It's actually MySQLDatabase.create / MySQLDatabase.delete. It's a static method on the Database class, not an instance method on the connection.
Keeping this issue open to track progress of the MySQLType protocols...
Closing in favor of #72, track there!
Thanks for the feeback @tanner0101 👍
Sorry for posting here again, but I want to get this to work so instead of waiting for MySQL support I switched from MySQL to PostgreSQL. Same structure as you proposed above but now using the PostgreSQLJSONType. Again those nested fields are missing from the schema 😓 Is there anything else involved with getting this to work?
import Foundation
//import FluentMySQL
import FluentPostgreSQL
import Vapor
enum MeasurementType: Int, Codable {
case placeA = 0
case placeB
case placeC
}
// Example JSON
// ------------
// {
// "type": 0,
// "measurement": {
// "temperature": 4.05,
// "ppressure": 77.81,
// "humidity": 45.23
// },
// "published_at": "2018-02-22T11:39:15.159Z"
// }
struct MeasurementData: PostgreSQLJSONType {
var temperature: Double
var pressure: Double
var humidity: Double
}
final class Measurement: Codable {
/// Model.ID
var id: Int?
var coreID: Core.ID?
var type: MeasurementType
var measurement: MeasurementData
var publishedAt: Date?
/// Timestampable.createdAt
var createdAt: Date?
/// Timestampable.updatedAt
var updatedAt: Date?
/// Initializer
init(id: ID? = nil, coreID: Core.ID?, type: MeasurementType, measurement: MeasurementData, publishedAt: Date) {
self.id = id
self.coreID = coreID
self.type = type
self.measurement = measurement
self.publishedAt = publishedAt
}
enum RootKeys: String, CodingKey {
case id
case coreID
case type
case measurement
case publishedAt = "published_at"
}
}table schema:
myTable-# \d+ measurements
Table "public.measurements"
Column | Type | Collation | Nullable | Default | Storage | Stats target | Description
-------------+-----------------------------+-----------+----------+----------------------------------+---------+--------------+-------------
id | bigint | | not null | generated by default as identity | plain | |
coreID | uuid | | | | plain | |
type | bigint | | not null | | plain | |
publishedAt | timestamp without time zone | | | | plain | |
createdAt | timestamp without time zone | | | | plain | |
updatedAt | timestamp without time zone | | | | plain | |
Indexes:
"measurements_pkey" PRIMARY KEY, btree (id)
It should work exactly as you have it there. Are you implementing a custom migration? It seems like some code is missing here. There's a discrepancy between published_at and the publishedAt field in the DB. That should use the coding key name (if it's properly configured). You should also need to conform MeasurementType to KeyStringDecodable. I'm not seeing that here.
(also feel free to reach out to me directly on http://vapor.team and I can help you in real time)
You're indeed right about that published_at :) I fixed that one in the meantime, the model still had RootKeys instead of CodingKeys, from the nestedContainer setup I had before. I have tried a number of things but it just does not create the JSONB column type... Model looks like this at the moment:
import Foundation
import FluentPostgreSQL
import Vapor
enum MeasurementType: Int, Codable {
case one = 0
case two
case three
case four
}
struct MeasurementData: PostgreSQLJSONType {
var temperature: Double
var pressure: Double
var humidity: Double
enum CodingKeys: String, CodingKey {
case temperature = "t"
case pressure = "p"
case humidity = "h"
}
}
// Example JSON
// ------------
// {
// "type": 0,
// "measurement": {
// "temperature": 4.05,
// "ppressure": 77.81,
// "humidity": 45.23
// },
// "published_at": "2018-02-22T11:39:15.159Z"
// }
final class Measurement: PostgreSQLModel {
/// Model.ID
static let idKey = \Measurement.id
var id: Int?
var type: MeasurementType
var measurement: MeasurementData
var publishedAt: Date?
/// Timestampable.createdAt
var createdAt: Date?
/// Timestampable.updatedAt
var updatedAt: Date?
/// Initializer
init(id: Int? = nil, type: MeasurementType, measurement: MeasurementData, publishedAt: Date) {
self.id = id
self.type = type
self.measurement = measurement
self.publishedAt = publishedAt
}
enum CodingKeys: String, CodingKey {
case id
case type
case measurement
case publishedAt = "published_at"
}
}
/// Stored date stamps
extension Measurement: Timestampable {
static var createdAtKey: WritableKeyPath<Measurement, Date?> {
return \.createdAt
}
static var updatedAtKey: WritableKeyPath<Measurement, Date?> {
return \.updatedAt
}
}
/// Support dynamic migrations.
extension Measurement: Migration { }
/// Support encoded to and decoded from HTTP messages.
extension Measurement: Content { }
/// Support using as a dynamic parameter in route definitions.
extension Measurement: Parameter { }pg_dump schema:
--
-- Name: measurements; Type: TABLE; Schema: public; Owner: jeroen
--
CREATE TABLE measurements (
id bigint NOT NULL,
type bigint NOT NULL,
published_at timestamp without time zone
);
ALTER TABLE measurements OWNER TO jeroen;
--
-- Name: measurements_id_seq; Type: SEQUENCE; Schema: public; Owner: jeroen
--
ALTER TABLE measurements ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY (
SEQUENCE NAME measurements_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1
);I'll reach out on Slack 👍
@twof this was related to an error in Core which has been fixed in 3.0.0 RC 1.0.1. I've added a test in PostgreSQL to make sure it's working correctly here on out
I'm having the same problem, but when i try to compile the code from 4np, i get: Use of undeclared type 'PostgreSQLJSONType'. Has it been replaced, or deleted?
@IgorRosocha it seems like PostgreSQLJSONType doesn't exist anymore.
But I found this article that saved my day in terms of saving objects to the database.