Tools: Nested Properties
Closed this issue · 4 comments
retendo commented
I would like to specify a nested input schema. It seems that is not possible at the moment.
Would you try to support that? I think there's a lot of use cases for this, when you want to let the model create structured data for you.
retendo commented
I use https://github.com/betalgo/openai for working with OpenAI endpoints and would like to at least use the same input schema for my function calls, to be able to compare them more easily.
retendo commented
What I would need is basically the last example on this documentation page (JSON mode): https://docs.anthropic.com/claude/docs/tool-use-examples
tghamm commented
Working on something like this:
public class ImageSchema
{
[JsonPropertyName("type")]
public string Type { get; set; } = "object";
[JsonPropertyName("required")]
public string[] Required { get; set; }
[JsonPropertyName("properties")]
public Properties Properties { get; set; }
}
public class Properties
{
[JsonPropertyName("key_colors")]
public KeyColorsProperty KeyColors { get; set; }
[JsonPropertyName("description")]
public DescriptionDetail Description { get; set; }
[JsonPropertyName("estimated_year")]
public EstimatedYear EstimatedYear { get; set; }
}
public class KeyColorsProperty
{
[JsonPropertyName("type")]
public string Type { get; set; } = "array";
[JsonPropertyName("items")]
public ItemProperty Items { get; set; }
[JsonPropertyName("description")]
public string Description { get; set; } = "Key colors in the image. Limit to less than four.";
}
public class DescriptionDetail
{
[JsonPropertyName("type")]
public string Type { get; set; } = "string";
[JsonPropertyName("description")]
public string Description { get; set; } = "Key colors in the image. Limit to less than four.";
}
public class EstimatedYear
{
[JsonPropertyName("type")]
public string Type { get; set; } = "string";
[JsonPropertyName("description")]
public string Description { get; set; } = "Estimated year that the images was taken, if is it a photo. Only set this if the image appears to be non-fictional. Rough estimates are okay!";
}
public class ItemProperty
{
public string Type { get; set; } = "object";
public Dictionary<string, ColorProperty> Properties { get; set; }
public List<string> Required { get; set; } = new List<string> { "r", "g", "b", "name" };
}
public class ColorProperty
{
[JsonPropertyName("type")]
public string Type { get; set; }
[JsonPropertyName("description")]
public string Description { get; set; }
}
[TestMethod]
public async Task TestClaude3ImageJsonModeMessage()
{
string resourceName = "Anthropic.SDK.Tests.Red_Apple.jpg";
Assembly assembly = Assembly.GetExecutingAssembly();
await using Stream stream = assembly.GetManifestResourceStream(resourceName);
byte[] imageBytes;
using (var memoryStream = new MemoryStream())
{
await stream.CopyToAsync(memoryStream);
imageBytes = memoryStream.ToArray();
}
string base64String = Convert.ToBase64String(imageBytes);
var client = new AnthropicClient();
var messages = new List<Message>();
messages.Add(new Message()
{
Role = RoleType.User,
Content = new dynamic[]
{
new ImageContent()
{
Source = new ImageSource()
{
MediaType = "image/jpeg",
Data = base64String
}
},
new TextContent()
{
Text = "Use `record_summary` to describe this image."
}
}
});
var imageSchema = new ImageSchema
{
Type = "object",
Required = new string[] { "key_colors", "description"},
Properties = new Properties()
{
KeyColors = new KeyColorsProperty
{
Items = new ItemProperty
{
Properties = new Dictionary<string, ColorProperty>
{
{ "r", new ColorProperty { Type = "number", Description = "red value [0.0, 1.0]" } },
{ "g", new ColorProperty { Type = "number", Description = "green value [0.0, 1.0]" } },
{ "b", new ColorProperty { Type = "number", Description = "blue value [0.0, 1.0]" } },
{ "name", new ColorProperty { Type = "string", Description = "Human-readable color name in snake_case, e.g. 'olive_green' or 'turquoise'" } }
}
}
},
Description = new DescriptionDetail { Type = "string", Description = "Image description. One to two sentences max." },
EstimatedYear = new EstimatedYear { Type = "number", Description = "Estimated year that the images was taken, if is it a photo. Only set this if the image appears to be non-fictional. Rough estimates are okay!" }
}
};
JsonSerializerOptions JsonSerializationOptions = new()
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
Converters = { new JsonStringEnumConverter() },
ReferenceHandler = ReferenceHandler.IgnoreCycles,
};
string jsonString = JsonSerializer.Serialize(imageSchema, JsonSerializationOptions);
var tools = new List<Common.Tool>()
{
new Common.Tool(new Function("record_summary", "Record summary of an image into well-structured JSON.",
JsonNode.Parse(jsonString)))
};
var parameters = new MessageParameters()
{
Messages = messages,
MaxTokens = 1024,
Model = AnthropicModels.Claude3Sonnet,
Stream = false,
Temperature = 1.0m,
};
var res = await client.Messages.GetClaudeMessageAsync(parameters, tools);
}
tghamm commented
Closing as complete with the latest release, feel free to open with any future issues.