Infinite restarts after startup
dohankk opened this issue · 4 comments
dohankk commented
problem
When I run it with docker compose, only the first data is synchronized, and the subsequent data seems to be out of sync.
And meilisync keeps re-running.
mysql log
2023-08-03 15:45:50 2023-08-03T06:45:50.279983Z 63 [Note] Aborted connection 63 to db: 'test' user: 'root' host: '172.22.0.4' (Got an error reading communication packets)
2023-08-03 15:45:50 2023-08-03T06:45:50.280026Z 64 [Note] Aborted connection 64 to db: 'test' user: 'root' host: '172.22.0.4' (Got an error reading communication packets)
meilisync log
2023-08-03 15:37:46 2023-08-03 06:37:46.050 | DEBUG | meilisync.main:_:33 - plugins=['meilisync.plugin.Plugin'] progress=Progress(type=<ProgressType.file: 'file'>) debug=True source=Source(type=<SourceType.mysql: 'mysql'>, database='test', user='root', host='mysql', password='1234', port=3306) meilisearch=MeiliSearch(api_url='http://meilisearch:7700', api_key='W_R0FiptNQ6f6CKCZPoGNGw4UuJfXc6j-B4YcVxwDw8', insert_size=1000, insert_interval=10) sync=[Sync(plugins=['meilisync.plugin.Plugin'], table='user', pk='id', full=True, index='user-collections', fields={'id': None, 'name': None, 'age': None, 'email': None})] sentry=None
2023-08-03 15:37:46 2023-08-03 06:37:46.092 | INFO | meilisync.main:_:82 - Full data sync for table "test.user" done! 2 documents added.
2023-08-03 15:37:46 2023-08-03 06:37:46.092 | INFO | meilisync.main:_:90 - Start increment sync data from "mysql" to MeiliSearch...
2023-08-03 15:37:46 ╭───────────────────── Traceback (most recent call last) ──────────────────────╮
2023-08-03 15:37:46 │ /meilisync/meilisync/main.py:129 in start │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ 126 │ │ lock = asyncio.Lock() │
2023-08-03 15:37:46 │ 127 │ │ await asyncio.gather(_(), interval()) │
2023-08-03 15:37:46 │ 128 │ │
2023-08-03 15:37:46 │ ❱ 129 │ asyncio.run(run()) │
2023-08-03 15:37:46 │ 130 │
2023-08-03 15:37:46 │ 131 │
2023-08-03 15:37:46 │ 132 @app.command(help="Delete all data in MeiliSearch and full sync") │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ ╭───────────────────────────────── locals ─────────────────────────────────╮ │
2023-08-03 15:37:46 │ │ _ = <function start.<locals>._ at 0x7f2f009abca0> │ │
2023-08-03 15:37:46 │ │ collection = <meilisync.event.EventCollection object at │ │
2023-08-03 15:37:46 │ │ 0x7f2f0093d6d0> │ │
2023-08-03 15:37:46 │ │ context = <click.core.Context object at 0x7f2f009a3e80> │ │
2023-08-03 15:37:46 │ │ current_progress = None │ │
2023-08-03 15:37:46 │ │ interval = <function start.<locals>.interval at 0x7f2f009abb80> │ │
2023-08-03 15:37:46 │ │ lock = <asyncio.locks.Lock object at 0x7f2f0093d3a0 │ │
2023-08-03 15:37:46 │ │ [unlocked]> │ │
2023-08-03 15:37:46 │ │ meili = <meilisync.meili.Meili object at 0x7f2f0093d3d0> │ │
2023-08-03 15:37:46 │ │ meili_settings = MeiliSearch( │ │
2023-08-03 15:37:46 │ │ │ api_url='http://meilisearch:7700', │ │
2023-08-03 15:37:46 │ │ │ │ │
2023-08-03 15:37:46 │ │ api_key='W_R0FiptNQ6f6CKCZPoGNGw4UuJfXc6j-B4YcVxwDw8… │ │
2023-08-03 15:37:46 │ │ │ insert_size=1000, │ │
2023-08-03 15:37:46 │ │ │ insert_interval=10 │ │
2023-08-03 15:37:46 │ │ ) │ │
2023-08-03 15:37:46 │ │ progress = <meilisync.progress.file.File object at │ │
2023-08-03 15:37:46 │ │ 0x7f2f0093d490> │ │
2023-08-03 15:37:46 │ │ run = <function start.<locals>.run at 0x7f2f009abc10> │ │
2023-08-03 15:37:46 │ │ settings = Settings( │ │
2023-08-03 15:37:46 │ │ │ plugins=['meilisync.plugin.Plugin'], │ │
2023-08-03 15:37:46 │ │ │ progress=Progress( │ │
2023-08-03 15:37:46 │ │ │ │ type=<ProgressType.file: 'file'> │ │
2023-08-03 15:37:46 │ │ │ ), │ │
2023-08-03 15:37:46 │ │ │ debug=True, │ │
2023-08-03 15:37:46 │ │ │ source=Source( │ │
2023-08-03 15:37:46 │ │ │ │ type=<SourceType.mysql: 'mysql'>, │ │
2023-08-03 15:37:46 │ │ │ │ database='test', │ │
2023-08-03 15:37:46 │ │ │ │ user='root', │ │
2023-08-03 15:37:46 │ │ │ │ host='mysql', │ │
2023-08-03 15:37:46 │ │ │ │ password='1234', │ │
2023-08-03 15:37:46 │ │ │ │ port=3306 │ │
2023-08-03 15:37:46 │ │ │ ), │ │
2023-08-03 15:37:46 │ │ │ meilisearch=MeiliSearch( │ │
2023-08-03 15:37:46 │ │ │ │ api_url='http://meilisearch:7700', │ │
2023-08-03 15:37:46 │ │ │ │ │ │
2023-08-03 15:37:46 │ │ api_key='W_R0FiptNQ6f6CKCZPoGNGw4UuJfXc6j-B4YcVxwDw8… │ │
2023-08-03 15:37:46 │ │ │ │ insert_size=1000, │ │
2023-08-03 15:37:46 │ │ │ │ insert_interval=10 │ │
2023-08-03 15:37:46 │ │ │ ), │ │
2023-08-03 15:37:46 │ │ │ sync=[ │ │
2023-08-03 15:37:46 │ │ │ │ Sync( │ │
2023-08-03 15:37:46 │ │ │ │ │ plugins=['meilisync.plugin.Plugin'], │ │
2023-08-03 15:37:46 │ │ │ │ │ table='user', │ │
2023-08-03 15:37:46 │ │ │ │ │ pk='id', │ │
2023-08-03 15:37:46 │ │ │ │ │ full=True, │ │
2023-08-03 15:37:46 │ │ │ │ │ index='user-collections', │ │
2023-08-03 15:37:46 │ │ │ │ │ fields={ │ │
2023-08-03 15:37:46 │ │ │ │ │ │ 'id': None, │ │
2023-08-03 15:37:46 │ │ │ │ │ │ 'name': None, │ │
2023-08-03 15:37:46 │ │ │ │ │ │ 'age': None, │ │
2023-08-03 15:37:46 │ │ │ │ │ │ 'email': None │ │
2023-08-03 15:37:46 │ │ │ │ │ } │ │
2023-08-03 15:37:46 │ │ │ │ ) │ │
2023-08-03 15:37:46 │ │ │ ], │ │
2023-08-03 15:37:46 │ │ │ sentry=None │ │
2023-08-03 15:37:46 │ │ ) │ │
2023-08-03 15:37:46 │ │ source = <meilisync.source.mysql.MySQL object at │ │
2023-08-03 15:37:46 │ │ 0x7f2f0093d8e0> │ │
2023-08-03 15:37:46 │ ╰──────────────────────────────────────────────────────────────────────────╯ │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ /usr/local/lib/python3.9/asyncio/runners.py:44 in run │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ 41 │ │ events.set_event_loop(loop) │
2023-08-03 15:37:46 │ 42 │ │ if debug is not None: │
2023-08-03 15:37:46 │ 43 │ │ │ loop.set_debug(debug) │
2023-08-03 15:37:46 │ ❱ 44 │ │ return loop.run_until_complete(main) │
2023-08-03 15:37:46 │ 45 │ finally: │
2023-08-03 15:37:46 │ 46 │ │ try: │
2023-08-03 15:37:46 │ 47 │ │ │ _cancel_all_tasks(loop) │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ ╭──────────────────────────────── locals ────────────────────────────────╮ │
2023-08-03 15:37:46 │ │ debug = None │ │
2023-08-03 15:37:46 │ │ loop = <_UnixSelectorEventLoop running=False closed=True debug=False> │ │
2023-08-03 15:37:46 │ │ main = <coroutine object start.<locals>.run at 0x7f2f009e3a40> │ │
2023-08-03 15:37:46 │ ╰────────────────────────────────────────────────────────────────────────╯ │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ /usr/local/lib/python3.9/asyncio/base_events.py:647 in run_until_complete │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ 644 │ │ if not future.done(): │
2023-08-03 15:37:46 │ 645 │ │ │ raise RuntimeError('Event loop stopped before Future comp │
2023-08-03 15:37:46 │ 646 │ │ │
2023-08-03 15:37:46 │ ❱ 647 │ │ return future.result() │
2023-08-03 15:37:46 │ 648 │ │
2023-08-03 15:37:46 │ 649 │ def stop(self): │
2023-08-03 15:37:46 │ 650 │ │ """Stop running the event loop. │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ ╭───────────────────────────────── locals ─────────────────────────────────╮ │
2023-08-03 15:37:46 │ │ future = <Task finished name='Task-4' coro=<start.<locals>.run() done, │ │
2023-08-03 15:37:46 │ │ defined at /meilisync/meilisync/main.py:124> │ │
2023-08-03 15:37:46 │ │ exception=TypeError("'NoneType' object is not │ │
2023-08-03 15:37:46 │ │ subscriptable")> │ │
2023-08-03 15:37:46 │ │ new_task = True │ │
2023-08-03 15:37:46 │ │ self = <_UnixSelectorEventLoop running=False closed=True │ │
2023-08-03 15:37:46 │ │ debug=False> │ │
2023-08-03 15:37:46 │ ╰──────────────────────────────────────────────────────────────────────────╯ │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ /meilisync/meilisync/main.py:127 in run │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ 124 │ async def run(): │
2023-08-03 15:37:46 │ 125 │ │ nonlocal lock │
2023-08-03 15:37:46 │ 126 │ │ lock = asyncio.Lock() │
2023-08-03 15:37:46 │ ❱ 127 │ │ await asyncio.gather(_(), interval()) │
2023-08-03 15:37:46 │ 128 │ │
2023-08-03 15:37:46 │ 129 │ asyncio.run(run()) │
2023-08-03 15:37:46 │ 130 │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ ╭────────────────────────────── locals ───────────────────────────────╮ │
2023-08-03 15:37:46 │ │ _ = <function start.<locals>._ at 0x7f2f009abca0> │ │
2023-08-03 15:37:46 │ │ interval = <function start.<locals>.interval at 0x7f2f009abb80> │ │
2023-08-03 15:37:46 │ │ lock = <asyncio.locks.Lock object at 0x7f2f0093d3a0 [unlocked]> │ │
2023-08-03 15:37:46 │ ╰─────────────────────────────────────────────────────────────────────╯ │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ /meilisync/meilisync/main.py:91 in _ │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ 88 │ │ │ │ │ │ │ f'No data found for table "{settings.sourc │
2023-08-03 15:37:46 │ 89 │ │ │ │ │ │ ) │
2023-08-03 15:37:46 │ 90 │ │ logger.info(f'Start increment sync data from "{settings.source │
2023-08-03 15:37:46 │ ❱ 91 │ │ async for event in source: │
2023-08-03 15:37:46 │ 92 │ │ │ if settings.debug: │
2023-08-03 15:37:46 │ 93 │ │ │ │ logger.debug(event) │
2023-08-03 15:37:46 │ 94 │ │ │ current_progress = event.progress │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ ╭───────────────────────────────── locals ─────────────────────────────────╮ │
2023-08-03 15:37:46 │ │ collection = <meilisync.event.EventCollection object at │ │
2023-08-03 15:37:46 │ │ 0x7f2f0093d6d0> │ │
2023-08-03 15:37:46 │ │ current_progress = None │ │
2023-08-03 15:37:46 │ │ data = [ │ │
2023-08-03 15:37:46 │ │ │ { │ │
2023-08-03 15:37:46 │ │ │ │ 'id': 1, │ │
2023-08-03 15:37:46 │ │ │ │ 'name': 'kim', │ │
2023-08-03 15:37:46 │ │ │ │ 'age': 11, │ │
2023-08-03 15:37:46 │ │ │ │ 'email': 't@gmail.com' │ │
2023-08-03 15:37:46 │ │ │ }, │ │
2023-08-03 15:37:46 │ │ │ { │ │
2023-08-03 15:37:46 │ │ │ │ 'id': 2, │ │
2023-08-03 15:37:46 │ │ │ │ 'name': 'lee', │ │
2023-08-03 15:37:46 │ │ │ │ 'age': 2, │ │
2023-08-03 15:37:46 │ │ │ │ 'email': 'tt@namver.com' │ │
2023-08-03 15:37:46 │ │ │ } │ │
2023-08-03 15:37:46 │ │ ] │ │
2023-08-03 15:37:46 │ │ lock = <asyncio.locks.Lock object at 0x7f2f0093d3a0 │ │
2023-08-03 15:37:46 │ │ [unlocked]> │ │
2023-08-03 15:37:46 │ │ meili = <meilisync.meili.Meili object at 0x7f2f0093d3d0> │ │
2023-08-03 15:37:46 │ │ meili_settings = MeiliSearch( │ │
2023-08-03 15:37:46 │ │ │ api_url='http://meilisearch:7700', │ │
2023-08-03 15:37:46 │ │ │ │ │
2023-08-03 15:37:46 │ │ api_key='W_R0FiptNQ6f6CKCZPoGNGw4UuJfXc6j-B4YcVxwDw8… │ │
2023-08-03 15:37:46 │ │ │ insert_size=1000, │ │
2023-08-03 15:37:46 │ │ │ insert_interval=10 │ │
2023-08-03 15:37:46 │ │ ) │ │
2023-08-03 15:37:46 │ │ progress = <meilisync.progress.file.File object at │ │
2023-08-03 15:37:46 │ │ 0x7f2f0093d490> │ │
2023-08-03 15:37:46 │ │ settings = Settings( │ │
2023-08-03 15:37:46 │ │ │ plugins=['meilisync.plugin.Plugin'], │ │
2023-08-03 15:37:46 │ │ │ progress=Progress( │ │
2023-08-03 15:37:46 │ │ │ │ type=<ProgressType.file: 'file'> │ │
2023-08-03 15:37:46 │ │ │ ), │ │
2023-08-03 15:37:46 │ │ │ debug=True, │ │
2023-08-03 15:37:46 │ │ │ source=Source( │ │
2023-08-03 15:37:46 │ │ │ │ type=<SourceType.mysql: 'mysql'>, │ │
2023-08-03 15:37:46 │ │ │ │ database='test', │ │
2023-08-03 15:37:46 │ │ │ │ user='root', │ │
2023-08-03 15:37:46 │ │ │ │ host='mysql', │ │
2023-08-03 15:37:46 │ │ │ │ password='1234', │ │
2023-08-03 15:37:46 │ │ │ │ port=3306 │ │
2023-08-03 15:37:46 │ │ │ ), │ │
2023-08-03 15:37:46 │ │ │ meilisearch=MeiliSearch( │ │
2023-08-03 15:37:46 │ │ │ │ api_url='http://meilisearch:7700', │ │
2023-08-03 15:37:46 │ │ │ │ │ │
2023-08-03 15:37:46 │ │ api_key='W_R0FiptNQ6f6CKCZPoGNGw4UuJfXc6j-B4YcVxwDw8… │ │
2023-08-03 15:37:46 │ │ │ │ insert_size=1000, │ │
2023-08-03 15:37:46 │ │ │ │ insert_interval=10 │ │
2023-08-03 15:37:46 │ │ │ ), │ │
2023-08-03 15:37:46 │ │ │ sync=[ │ │
2023-08-03 15:37:46 │ │ │ │ Sync( │ │
2023-08-03 15:37:46 │ │ │ │ │ plugins=['meilisync.plugin.Plugin'], │ │
2023-08-03 15:37:46 │ │ │ │ │ table='user', │ │
2023-08-03 15:37:46 │ │ │ │ │ pk='id', │ │
2023-08-03 15:37:46 │ │ │ │ │ full=True, │ │
2023-08-03 15:37:46 │ │ │ │ │ index='user-collections', │ │
2023-08-03 15:37:46 │ │ │ │ │ fields={ │ │
2023-08-03 15:37:46 │ │ │ │ │ │ 'id': None, │ │
2023-08-03 15:37:46 │ │ │ │ │ │ 'name': None, │ │
2023-08-03 15:37:46 │ │ │ │ │ │ 'age': None, │ │
2023-08-03 15:37:46 │ │ │ │ │ │ 'email': None │ │
2023-08-03 15:37:46 │ │ │ │ │ } │ │
2023-08-03 15:37:46 │ │ │ │ ) │ │
2023-08-03 15:37:46 │ │ │ ], │ │
2023-08-03 15:37:46 │ │ │ sentry=None │ │
2023-08-03 15:37:46 │ │ ) │ │
2023-08-03 15:37:46 │ │ source = <meilisync.source.mysql.MySQL object at │ │
2023-08-03 15:37:46 │ │ 0x7f2f0093d8e0> │ │
2023-08-03 15:37:46 │ │ sync = Sync( │ │
2023-08-03 15:37:46 │ │ │ plugins=['meilisync.plugin.Plugin'], │ │
2023-08-03 15:37:46 │ │ │ table='user', │ │
2023-08-03 15:37:46 │ │ │ pk='id', │ │
2023-08-03 15:37:46 │ │ │ full=True, │ │
2023-08-03 15:37:46 │ │ │ index='user-collections', │ │
2023-08-03 15:37:46 │ │ │ fields={ │ │
2023-08-03 15:37:46 │ │ │ │ 'id': None, │ │
2023-08-03 15:37:46 │ │ │ │ 'name': None, │ │
2023-08-03 15:37:46 │ │ │ │ 'age': None, │ │
2023-08-03 15:37:46 │ │ │ │ 'email': None │ │
2023-08-03 15:37:46 │ │ │ } │ │
2023-08-03 15:37:46 │ │ ) │ │
2023-08-03 15:37:46 │ ╰──────────────────────────────────────────────────────────────────────────╯ │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ /meilisync/meilisync/source/mysql.py:67 in __aiter__ │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ 64 │ │ │ master_log_file = self.progress["master_log_file"] │
2023-08-03 15:37:46 │ 65 │ │ │ master_log_position = int(self.progress["master_log_positi │
2023-08-03 15:37:46 │ 66 │ │ else: │
2023-08-03 15:37:46 │ ❱ 67 │ │ │ master_log_file, master_log_position = await self._get_bin │
2023-08-03 15:37:46 │ 68 │ │ yield ProgressEvent( │
2023-08-03 15:37:46 │ 69 │ │ │ progress={ │
2023-08-03 15:37:46 │ 70 │ │ │ │ "master_log_file": master_log_file, │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ ╭──────────────────────────── locals ────────────────────────────╮ │
2023-08-03 15:37:46 │ │ self = <meilisync.source.mysql.MySQL object at 0x7f2f0093d8e0> │ │
2023-08-03 15:37:46 │ ╰────────────────────────────────────────────────────────────────╯ │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ /meilisync/meilisync/source/mysql.py:58 in _get_binlog_position │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ 55 │ │ async with self.conn.cursor(cursor=DictCursor) as cur: │
2023-08-03 15:37:46 │ 56 │ │ │ await cur.execute("SHOW MASTER STATUS") │
2023-08-03 15:37:46 │ 57 │ │ │ ret = await cur.fetchone() │
2023-08-03 15:37:46 │ ❱ 58 │ │ │ return ret["File"], ret["Position"] │
2023-08-03 15:37:46 │ 59 │ │
2023-08-03 15:37:46 │ 60 │ async def __aiter__(self): │
2023-08-03 15:37:46 │ 61 │ │ self.conn = await asyncmy.connect(**self.kwargs) │
2023-08-03 15:37:46 │ │
2023-08-03 15:37:46 │ ╭──────────────────────────── locals ────────────────────────────╮ │
2023-08-03 15:37:46 │ │ cur = <asyncmy.cursors.DictCursor object at 0x7f2f000ef160> │ │
2023-08-03 15:37:46 │ │ ret = None │ │
2023-08-03 15:37:46 │ │ self = <meilisync.source.mysql.MySQL object at 0x7f2f0093d8e0> │ │
2023-08-03 15:37:46 │ ╰────────────────────────────────────────────────────────────────╯ │
2023-08-03 15:37:46 ╰──────────────────────────────────────────────────────────────────────────────╯
2023-08-03 15:37:46 TypeError: 'NoneType' object is not subscriptable
docker-compose.yaml
version: '3'
services:
mysql:
image: mysql:5
container_name: mysql
restart: always
platform: linux/amd64
environment:
MYSQL_ROOT_PASSWORD: "1234"
MYSQL_DATABASE: test
MYSQL_USER: kim
MYSQL_PASSWORD: "1234"
healthcheck:
test:
["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-p1234"]
interval: 30s
timeout: 10s
retries: 5
volumes:
- ./init.sql:/docker-entrypoint-initdb.d/init.sql
ports:
- 3306:3306
meilisearch:
container_name: meilisearch
image: getmeili/meilisearch:v1.2
environment:
- http_proxy
- https_proxy
- MEILI_MASTER_KEY=W_R0FiptNQ6f6CKCZPoGNGw4UuJfXc6j-B4YcVxwDw8
# - MEILI_MASTER_KEY=${MEILI_MASTER_KEY:-masterKey}
- MEILI_NO_ANALYTICS=${MEILI_NO_ANALYTICS:-true}
- MEILI_ENV=${MEILI_ENV:-development}
- MEILI_LOG_LEVEL
- MEILI_DB_PATH=${MEILI_DB_PATH:-/data.ms}
ports:
- ${MEILI_PORT:-7700}:7700
volumes:
- ./data.ms:/data.ms
restart: unless-stopped
depends_on:
mysql:
condition: service_healthy
meilisync:
image: long2ice/meilisync
volumes:
- ./config.yml:/meilisync/config.yml
restart: always
depends_on:
- meilisearch
config.yaml
debug: true
plugins:
- meilisync.plugin.Plugin
progress:
type: file
source:
type: mysql
host: mysql
port: 3306
user: root
password: "1234"
database: test
meilisearch:
api_url: http://meilisearch:7700
api_key: W_R0FiptNQ6f6CKCZPoGNGw4UuJfXc6j-B4YcVxwDw8
insert_size: 1000
insert_interval: 10
sync:
- table: user
index: user-collections
plugins:
- meilisync.plugin.Plugin
full: true
fields:
id:
name:
age:
email:
# sentry:
# dsn: ""
# environment: "production"
long2ice commented
Try mysql8, I did not tested it in mysql5
dohankk commented
It works fine in version 8.
I don't know why it doesn't work in version 5.
dohankk commented
If I set insert_size: 1000
but not insert_interval
, will it not synchronize?
long2ice commented
Only one of them needs to be satisfied