-- Table to store information about different models
CREATE TABLE Models (
ModelID INT PRIMARY KEY,
ModelName VARCHAR(255) NOT NULL
);
-- Table to store information about various capabilities
CREATE TABLE Capabilities (
CapabilityID INT PRIMARY KEY,
CapabilityName VARCHAR(255) NOT NULL
);
-- Table to store benchmark scores for different models and capabilities
CREATE TABLE Benchmarks (
BenchmarkID INT PRIMARY KEY,
ModelID INT,
CapabilityID INT,
BenchmarkName VARCHAR(255) NOT NULL,
ScoreGemini FLOAT,
ScoreGPT4 FLOAT,
Description TEXT,
FOREIGN KEY (ModelID) REFERENCES Models(ModelID),
FOREIGN KEY (CapabilityID) REFERENCES Capabilities(CapabilityID)
);
-- Insert data into the Models table
INSERT INTO Models (ModelID, ModelName) VALUES
(1, 'Gemini Ultra'),
(2, 'GPT-4');
-- Insert data into the Capabilities table
INSERT INTO Capabilities (CapabilityID, CapabilityName) VALUES
(1, 'General'),
(2, 'Reasoning'),
(3, 'Math'),
(4, 'Code'),
(5, 'Image'),
(6, 'Video'),
(7, 'Audio');
-- Insert data into the Benchmarks table
INSERT INTO Benchmarks (BenchmarkID, ModelID, CapabilityID, BenchmarkName, ScoreGemini, ScoreGPT4, Description) VALUES
-- General Capabilities
(1, 1, 1, 'MMLU', 90.00, 86.40, 'Representation of questions in 57 subjects'),
(2, 2, 1, 'MMLU', 86.40, NULL, 'Representation of questions in 57 subjects'),
-- Reasoning Capabilities
(3, 1, 2, 'Big-Bench Hard', 83.60, 83.10, 'Diverse set of challenging tasks requiring multi-step reasoning'),
(4, 2, 2, 'Big-Bench Hard', 83.10, NULL, 'Diverse set of challenging tasks requiring multi-step reasoning'),
(5, 1, 2, 'DROP', 82.4, 80.9, 'Reading comprehension (Fl Score)'),
(6, 2, 2, 'DROP', 80.9, NULL, 'Reading comprehension (Fl Score)'),
(7, 1, 2, 'HellaSwag', 87.80, 95.30, 'Commonsense reasoning for everyday tasks'),
(8, 2, 2, 'HellaSwag', 95.30, NULL, 'Commonsense reasoning for everyday tasks'),
-- Math Capabilities
(9, 1, 3, 'GSM8K', 94.40, 92.00, 'Basic arithmetic manipulations, incl. Grade School math problems'),
(10, 2, 3, 'GSM8K', 92.00, NULL, 'Basic arithmetic manipulations, incl. Grade School math problems'),
(11, 1, 3, 'MATH', 53.20, 52.90, 'Challenging math problems, incl. algebra, geometry, pre-calculus, and others'),
(12, 2, 3, 'MATH', 52.90, NULL, 'Challenging math problems, incl. algebra, geometry, pre-calculus, and others')
### Questions 🤔
1. What are the average scores for each capability on both the Gemini Ultra and GPT-4 models?
2. Which benchmarks does Gemini Ultra outperform GPT-4 in terms of scores?
3. What are the highest scores achieved by Gemini Ultra and GPT-4 for each benchmark in the Image capability?
4. Calculate the percentage improvement of Gemini Ultra over GPT-4 for each benchmark?
5. Retrieve the benchmarks where both models scored above the average for their respective models?
6. Which benchmarks show that Gemini Ultra is expected to outperform GPT-4 based on the next score?
7. Classify benchmarks into performance categories based on score ranges?
8. Retrieve the rankings for each capability based on Gemini Ultra scores?
9. Convert the Capability and Benchmark names to uppercase?
10. Can you provide the benchmarks along with their descriptions in a concatenated format?
Feel free to use the SQL queries provided to derive the answers. Let's explore the performance and capabilities of Gemini Ultra and GPT-4 models!