Missing nori analyzer
Opened this issue · 9 comments
i have sourc code pluginType org.test.plugin.analysis.TestPlugin
org.test.plugin.analysis.TestPlugin has the actual plugin source, so why not?
runner = new ElasticsearchClusterRunner();
runner.onBuild(new ElasticsearchClusterRunner.Builder() {
@Override
public void build(final int number, final Settings.Builder settingsBuilder) {
settingsBuilder.put("http.cors.enabled", true);
settingsBuilder.put("http.cors.allow-origin", "*");
settingsBuilder.put("discovery.type", "single-node");
// settingsBuilder.putList("discovery.seed_hosts", "127.0.0.1:9301");
// settingsBuilder.putList("cluster.initial_master_nodes", "127.0.0.1:9301");
}
}).build(new ElasticsearchClusterRunner.Configs().clusterName(clusterName).numOfNode(numOfNode).pluginTypes("org.test.plugin.analysis.TestPlugin"));
mappings.json
{
"settings": {
"index": {
"analysis": {
"tokenizer": {
"korean_nori_tokenizer": {
"type": "reloadable_nori_tokenizer",
"decompound_mode": "discard"
}
},
"analyzer": {
"my_nori_analyzer": {
"tokenizer": "korean_nori_tokenizer",
"filter": [
"my_nori_filter"
]
}
},
"filter": {
"my_nori_filter": {
"type": "nori_part_of_speech",
"stoptags": [
"SP"
],
"updateable": true
}
}
}
}
},
"mappings": {
"properties": {
"test": {
"type": "keyword",
"fields": {
"nori": {
"search_analyzer": "my_nori_analyzer",
"analyzer": "my_nori_analyzer",
"type": "text"
}
}
}
}
}
}
[2022-01-10T20:37:13,949][INFO ][o.e.p.PluginsService ] loaded plugin [org.elasticsearch.analysis.common.CommonAnalysisPlugin]
[2022-01-10T20:37:13,950][INFO ][o.e.p.PluginsService ] loaded plugin [org.elasticsearch.index.mapper.MapperExtrasPlugin]
[2022-01-10T20:37:13,950][INFO ][o.e.p.PluginsService ] loaded plugin [org.elasticsearch.index.rankeval.RankEvalPlugin]
[2022-01-10T20:37:13,950][INFO ][o.e.p.PluginsService ] loaded plugin [org.elasticsearch.index.reindex.ReindexPlugin]
[2022-01-10T20:37:13,950][INFO ][o.e.p.PluginsService ] loaded plugin [org.elasticsearch.ingest.common.IngestCommonPlugin]
[2022-01-10T20:37:13,950][INFO ][o.e.p.PluginsService ] loaded plugin [org.elasticsearch.ingest.useragent.IngestUserAgentPlugin]
[2022-01-10T20:37:13,951][INFO ][o.e.p.PluginsService ] loaded plugin [org.elasticsearch.join.ParentJoinPlugin]
[2022-01-10T20:37:13,951][INFO ][o.e.p.PluginsService ] loaded plugin [org.elasticsearch.painless.PainlessPlugin]
[2022-01-10T20:37:13,951][INFO ][o.e.p.PluginsService ] loaded plugin [org.elasticsearch.percolator.PercolatorPlugin]
[2022-01-10T20:37:13,951][INFO ][o.e.p.PluginsService ] loaded plugin [org.elasticsearch.plugin.analysis. TestPlugin]
[2022-01-10T20:37:13,952][INFO ][o.e.p.PluginsService ] loaded plugin [org.elasticsearch.plugin.repository.url.URLRepositoryPlugin]
[2022-01-10T20:37:13,952][INFO ][o.e.p.PluginsService ] loaded plugin [org.elasticsearch.script.expression.ExpressionPlugin]
[2022-01-10T20:37:13,952][INFO ][o.e.p.PluginsService ] loaded plugin [org.elasticsearch.script.mustache.MustachePlugin]
[2022-01-10T20:37:13,952][INFO ][o.e.p.PluginsService ] loaded plugin [org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin]
[2022-01-10T20:37:13,952][INFO ][o.e.p.PluginsService ] loaded plugin [org.elasticsearch.transport.Netty4Plugin]
unknown setting [index.mappings.properties.test.fields.nori.analyzer] please check that any required plugins are installed, or check the breaking changes documentation for removed settings
java.lang.IllegalArgumentException: unknown setting [index.mappings.properties.test.fields.nori.analyzer] please check that any required plugins are installed, or check the breaking changes documentation for removed settings
at org.elasticsearch.common.settings.AbstractScopedSettings.validate(AbstractScopedSettings.java:544)
at org.elasticsearch.common.settings.AbstractScopedSettings.validate(AbstractScopedSettings.java:489)
at org.elasticsearch.common.settings.AbstractScopedSettings.validate(AbstractScopedSettings.java:460)
at org.elasticsearch.common.settings.AbstractScopedSettings.validate(AbstractScopedSettings.java:431)
at org.elasticsearch.cluster.metadata.MetadataCreateIndexService.normalizeRequestSetting(MetadataCreateIndexService.java:319)
at org.elasticsearch.cluster.metadata.MetadataCreateIndexService.onlyCreateIndex(MetadataCreateIndexService.java:289)
at org.elasticsearch.cluster.metadata.MetadataCreateIndexService.createIndex(MetadataCreateIndexService.java:271)
at org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction.masterOperation(TransportCreateIndexAction.java:86)
at org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction.masterOperation(TransportCreateIndexAction.java:41)
at org.elasticsearch.action.support.master.TransportMasterNodeAction.masterOperation(TransportMasterNodeAction.java:99)
at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.lambda$doStart$3(TransportMasterNodeAction.java:166)
at org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:224)
at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.doStart(TransportMasterNodeAction.java:166)
at org.elasticsearch.action.support.master.TransportMasterNodeAction.doExecute(TransportMasterNodeAction.java:115)
at org.elasticsearch.action.support.master.TransportMasterNodeAction.doExecute(TransportMasterNodeAction.java:59)
at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:179)
at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:155)
at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:83)
at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:86)
at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:75)
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:412)
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:401)
at org.elasticsearch.client.support.AbstractClient$IndicesAdmin.execute(AbstractClient.java:1286)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:45)
at org.codelibs.elasticsearch.runner.ElasticsearchClusterRunner.createIndex(ElasticsearchClusterRunner.java:821)
at org.codelibs.elasticsearch.runner.ElasticsearchClusterRunner.createIndex(ElasticsearchClusterRunner.java:817)
at org.elasticsearch.plugin.analyzerTest.ReloadableNoriTokenizerFactoryTest.test_basic(ReloadableNoriTokenizerFactoryTest.java:124)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:567)
at junit.framework.TestCase.runTest(TestCase.java:176)
at junit.framework.TestCase.runBare(TestCase.java:141)
at junit.framework.TestResult$1.protect(TestResult.java:122)
at junit.framework.TestResult.runProtected(TestResult.java:142)
at junit.framework.TestResult.run(TestResult.java:125)
at junit.framework.TestCase.run(TestCase.java:129)
at junit.framework.TestSuite.runTest(TestSuite.java:252)
at junit.framework.TestSuite.run(TestSuite.java:247)
at org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:86)
at org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.runTestClass(JUnitTestClassExecutor.java:110)
at org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:58)
at org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:38)
at org.gradle.api.internal.tasks.testing.junit.AbstractJUnitTestClassProcessor.processTestClass(AbstractJUnitTestClassProcessor.java:62)
at org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:51)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:567)
at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
at org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33)
at org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94)
at com.sun.proxy.$Proxy2.processTestClass(Unknown Source)
at org.gradle.api.internal.tasks.testing.worker.TestWorker.processTestClass(TestWorker.java:118)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:567)
at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36)
at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24)
at org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:182)
at org.gradle.internal.remote.internal.hub.MessageHubBackedObjectConnection$DispatchWrapper.dispatch(MessageHubBackedObjectConnection.java:164)
at org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:412)
at org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64)
at org.gradle.internal.concurrent.ManagedExecutorImpl$1.run(ManagedExecutorImpl.java:48)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at org.gradle.internal.concurrent.ThreadFactoryImpl$ManagedThreadRunnable.run(ThreadFactoryImpl.java:56)
at java.base/java.lang.Thread.run(Thread.java:830)
Suppressed: java.lang.IllegalArgumentException: unknown setting [index.mappings.properties.test.fields.nori.search_analyzer] please check that any required plugins are installed, or check the breaking changes documentation for removed settings
... 72 more
Suppressed: java.lang.IllegalArgumentException: unknown setting [index.mappings.properties.test.fields.nori.type] please check that any required plugins are installed, or check the breaking changes documentation for removed settings
... 72 more
Suppressed: java.lang.IllegalArgumentException: unknown setting [index.mappings.properties.test.type] please check that any required plugins are installed, or check the breaking changes documentation for removed settings
... 72 more
Suppressed: java.lang.IllegalArgumentException: unknown setting [index.settings.index.analysis.analyzer.my_nori_analyzer.filter] please check that any required plugins are installed, or check the breaking changes documentation for removed settings
... 72 more
Suppressed: java.lang.IllegalArgumentException: unknown setting [index.settings.index.analysis.analyzer.my_nori_analyzer.tokenizer] please check that any required plugins are installed, or check the breaking changes documentation for removed settings
... 72 more
Suppressed: java.lang.IllegalArgumentException: unknown setting [index.settings.index.analysis.filter.my_nori_filter.stoptags] please check that any required plugins are installed, or check the breaking changes documentation for removed settings
... 72 more
Suppressed: java.lang.IllegalArgumentException: unknown setting [index.settings.index.analysis.filter.my_nori_filter.type] please check that any required plugins are installed, or check the breaking changes documentation for removed settings
... 72 more
Suppressed: java.lang.IllegalArgumentException: unknown setting [index.settings.index.analysis.filter.my_nori_filter.updateable] please check that any required plugins are installed, or check the breaking changes documentation for removed settings
... 72 more
Suppressed: java.lang.IllegalArgumentException: unknown setting [index.settings.index.analysis.tokenizer.korean_nori_tokenizer.decompound_mode] please check that any required plugins are installed, or check the breaking changes documentation for removed settings
... 72 more
Suppressed: java.lang.IllegalArgumentException: unknown setting [index.settings.index.analysis.tokenizer.korean_nori_tokenizer.type] please check that any required plugins are installed, or check the breaking changes documentation for removed settings
... 72 more
Suppressed: java.lang.IllegalArgumentException: unknown setting [index.settings.index.analysis.tokenizer.korean_nori_tokenizer.user_dictionary] please check that any required plugins are installed, or check the breaking changes documentation for removed settings
... 72 more
The plugin is actually loaded, but I keep getting an error
Did you add nori?
@marevol
yes i have a nori
@Override
public Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> extra = new HashMap<>();
extra.put("nori_part_of_speech", NoriPartOfSpeechStopFilterFactory::new);
extra.put("nori_readingform", NoriReadingFormFilterFactory::new);
extra.put("nori_number", NoriNumberFilterFactory::new);
return extra;
}
@Override
public Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> getTokenizers() {
final Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> extra = new HashMap();
extra.put("reloadable_nori_tokenizer", ReloadableNoriTokenizerFactory::new);
extra.put("nori_tokenizer", NoriTokenizerFactory::new);
return extra;
}
@marevol
I know you are very busy, but please ask when you can!
How did you send the mappings.json?
@marevol
For reference, there was no problem when creating an index directly on es.
thank u
{
"settings": {
"index": {
"analysis": {
"tokenizer": {
"korean_nori_tokenizer": {
"type": "reloadable_nori_tokenizer",
"decompound_mode": "discard"
}
},
"analyzer": {
"my_nori_analyzer": {
"tokenizer": "korean_nori_tokenizer",
"filter": [
"my_nori_filter"
]
}
},
"filter": {
"my_nori_filter": {
"type": "nori_part_of_speech",
"stoptags": [
"SP"
],
"updateable": true
}
}
}
}
},
"mappings": {
"properties": {
"test": {
"type": "keyword",
"fields": {
"nori": {
"search_analyzer": "my_nori_analyzer",
"analyzer": "my_nori_analyzer",
"type": "text"
}
}
}
}
}
}
Could you provide information to reproduce it?
package org.elasticsearch.plugin.analyzerTest;
import java.io.File;
import junit.framework.TestCase;
import org.codelibs.elasticsearch.runner.ElasticsearchClusterRunner;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.node.Node;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class ReloadableNoriTokenizerFactoryTest extends TestCase {
private ElasticsearchClusterRunner runner;
private int numOfNode = 1;
private File[] keywordFiles;
private String clusterName;
@Before
public void setUp() throws Exception {
clusterName = "es-analysisja-" + System.currentTimeMillis();
runner = new ElasticsearchClusterRunner();
runner.onBuild(new ElasticsearchClusterRunner.Builder() {
@Override
public void build(final int number, final Settings.Builder settingsBuilder) {
settingsBuilder.put("http.cors.enabled", true);
settingsBuilder.put("http.cors.allow-origin", "*");
settingsBuilder.put("discovery.type", "single-node");
// settingsBuilder.putList("discovery.seed_hosts", "127.0.0.1:9301");
// settingsBuilder.putList("cluster.initial_master_nodes", "127.0.0.1:9301");
}
}).build(new ElasticsearchClusterRunner.Configs().clusterName(clusterName).numOfNode(numOfNode));
keywordFiles = null;
}
@After
public void cleanUp() throws Exception {
runner.close();
runner.clean();
if (keywordFiles != null) {
for (File file : keywordFiles) {
file.deleteOnExit();
}
}
}
@Test
public void test_basic() throws Exception {
runner.ensureYellow();
Node node = runner.node();
final String index = "dataset";
final String indexSettings = "{\n" +
"" +
" \"mappings\": {\n" +
" \"properties\": {\n" +
" \"hahahoho\": {\n" +
" \"type\": \"keyword\"\n" +
" }\n" +
" }\n" +
" }\n" +
"}";
runner.createIndex(index, Settings.builder().loadFromSource(indexSettings, XContentType.JSON).build());
runner.ensureYellow();
}
}
I tried with a simple code, but this doesn't work either
Am I unable to load the default plugin?
unknown setting [index.mappings.properties.hahahoho.type] please check that any required plugins are installed, or check the breaking changes documentation for removed settings
java.lang.IllegalArgumentException: unknown setting [index.mappings.properties.hahahoho.type] please check that any required plugins are installed, or check the breaking changes documentation for removed settings
It's for an index setting, not mapping.
final String indexSettings = "{\n" +
" \"properties\": {\n" +
" \"hahahoho\": {\n" +
" \"type\": \"keyword\"\n" +
" }\n" +
" }\n" +
"}";
runner.createIndex(index, builder -> builder.addMapping("_doc",
indexSettings, XContentType.JSON));
@marevol
From what you tested elsewhere, I think you should remove the settings property and set it up.
Is it impossible to test until data is mapped and retrieved?