mars-project/mars

[BUG] Ray executor raises ValueError: WRITEBACKIFCOPY base is read-only

fyrestone opened this issue · 0 comments

fyrestone commented

Describe the bug
A clear and concise description of what the bug is.

_____________________ test_predict_sparse_callable_kernel ______________________

setup = <mars.deploy.oscar.session.SyncSession object at 0x33564eee0>

    def test_predict_sparse_callable_kernel(setup):
        # This is a non-regression test for #15866
    
        # Custom sparse kernel (top-K RBF)
        def topk_rbf(X, Y=None, n_neighbors=10, gamma=1e-5):
            nn = NearestNeighbors(n_neighbors=10, metric="euclidean", n_jobs=-1)
            nn.fit(X)
            W = -1 * mt.power(nn.kneighbors_graph(Y, mode="distance"), 2) * gamma
            W = mt.exp(W)
            assert W.issparse()
            return W.T
    
        n_classes = 4
        n_samples = 500
        n_test = 10
        X, y = make_classification(
            n_classes=n_classes,
            n_samples=n_samples,
            n_features=20,
            n_informative=20,
            n_redundant=0,
            n_repeated=0,
            random_state=0,
        )
    
        X_train, X_test, y_train, y_test = train_test_split(
            X, y, test_size=n_test, random_state=0
        )
    
        model = LabelPropagation(kernel=topk_rbf)
>       model.fit(X_train, y_train)

mars/learn/semi_supervised/tests/test_label_propagation.py:143: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
mars/learn/semi_supervised/_label_propagation.py:369: in fit
    return super().fit(X, y, session=session, run_kwargs=run_kwargs)
mars/learn/semi_supervised/_label_propagation.py:231: in fit
    ExecutableTuple(to_run).execute(session=session, **(run_kwargs or dict()))
mars/core/entity/executable.py:267: in execute
    ret = execute(*self, session=session, **kw)
mars/deploy/oscar/session.py:1888: in execute
    return session.execute(
mars/deploy/oscar/session.py:1682: in execute
    execution_info: ExecutionInfo = fut.result(
../../.pyenv/versions/3.8.13/lib/python3.8/concurrent/futures/_base.py:444: in result
    return self.__get_result()
../../.pyenv/versions/3.8.13/lib/python3.8/concurrent/futures/_base.py:389: in __get_result
    raise self._exception
mars/deploy/oscar/session.py:1868: in _execute
    await execution_info
../../.pyenv/versions/3.8.13/lib/python3.8/asyncio/tasks.py:695: in _wrap_awaitable
    return (yield from awaitable.__await__())
mars/deploy/oscar/session.py:105: in wait
    return await self._aio_task
mars/deploy/oscar/session.py:953: in _run_in_background
    raise task_result.error.with_traceback(task_result.traceback)
mars/services/task/supervisor/processor.py:372: in run
    await self._process_stage_chunk_graph(*stage_args)
mars/services/task/supervisor/processor.py:250: in _process_stage_chunk_graph
    chunk_to_result = await self._executor.execute_subtask_graph(
mars/services/task/execution/ray/executor.py:551: in execute_subtask_graph
    meta_list = await asyncio.gather(*output_meta_object_refs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

awaitable = ObjectRef(c3f6db450a565c05ffffffffffffffffffffffff0100000001000000)

    @types.coroutine
    def _wrap_awaitable(awaitable):
        """Helper for asyncio.ensure_future().
    
        Wraps awaitable (an object with __await__) into a coroutine
        that will later be wrapped in a Task by ensure_future().
        """
>       return (yield from awaitable.__await__())
E       ray.exceptions.RayTaskError(ValueError): ray::execute_subtask() (pid=15135, ip=127.0.0.1)
E         At least one of the input arguments for this task could not be computed:
E       ray.exceptions.RayTaskError: ray::execute_subtask() (pid=15135, ip=127.0.0.1)
E         At least one of the input arguments for this task could not be computed:
E       ray.exceptions.RayTaskError: ray::execute_subtask() (pid=15135, ip=127.0.0.1)
E         File "/home/admin/mars/mars/services/task/execution/ray/executor.py", line 185, in execute_subtask
E           execute(context, chunk.op)
E         File "/home/admin/mars/mars/core/operand/core.py", line 491, in execute
E           result = executor(results, op)
E         File "/home/admin/mars/mars/tensor/arithmetic/core.py", line 165, in execute
E           ret = cls._execute_cpu(op, xp, lhs, rhs, **kw)
E         File "/home/admin/mars/mars/tensor/arithmetic/core.py", line 142, in _execute_cpu
E           return cls._get_func(xp)(lhs, rhs, **kw)
E         File "/home/admin/mars/mars/lib/sparse/__init__.py", line 93, in power
E           return a**b
E         File "/home/admin/mars/mars/lib/sparse/array.py", line 503, in __pow__
E           x = self.spmatrix.power(naked_other)
E         File "/home/admin/.pyenv/versions/3.8.13/lib/python3.8/site-packages/scipy/sparse/_data.py", line 114, in power
E           data = self._deduped_data()
E         File "/home/admin/.pyenv/versions/3.8.13/lib/python3.8/site-packages/scipy/sparse/_data.py", line 32, in _deduped_data
E           self.sum_duplicates()
E         File "/home/admin/.pyenv/versions/3.8.13/lib/python3.8/site-packages/scipy/sparse/_compressed.py", line 1118, in sum_duplicates
E           self.sort_indices()
E         File "/home/admin/.pyenv/versions/3.8.13/lib/python3.8/site-packages/scipy/sparse/_compressed.py", line 1164, in sort_indices
E           _sparsetools.csr_sort_indices(len(self.indptr) - 1, self.indptr,
E       ValueError: WRITEBACKIFCOPY base is read-only

../../.pyenv/versions/3.8.13/lib/python3.8/asyncio/tasks.py:695: RayTaskError(ValueError)
________________________ test_label_binarize_multilabel ________________________

setup = <mars.deploy.oscar.session.SyncSession object at 0x332666190>

    def test_label_binarize_multilabel(setup):
        y_ind = np.array([[0, 1, 0], [1, 1, 1], [0, 0, 0]])
        classes = [0, 1, 2]
        pos_label = 2
        neg_label = 0
        expected = pos_label * y_ind
        y_sparse = [sp.csr_matrix(y_ind)]
    
        for y in [y_ind] + y_sparse:
>           check_binarized_results(y, classes, pos_label, neg_label, expected)

mars/learn/preprocessing/tests/test_label.py:250: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
mars/learn/preprocessing/tests/test_label.py:186: in check_binarized_results
    inversed = _inverse_binarize_thresholding(
../../.pyenv/versions/3.8.13/lib/python3.8/site-packages/sklearn/preprocessing/_label.py:649: in _inverse_binarize_thresholding
    y.eliminate_zeros()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <3x3 sparse matrix of type '<class 'numpy.int64'>'
	with 4 stored elements in Compressed Sparse Row format>

    def eliminate_zeros(self):
        """Remove zero entries from the matrix
    
        This is an *in place* operation.
        """
        M, N = self._swap(self.shape)
>       _sparsetools.csr_eliminate_zeros(M, N, self.indptr, self.indices,
                                         self.data)
E       ValueError: WRITEBACKIFCOPY base is read-only

../../.pyenv/versions/3.8.13/lib/python3.8/site-packages/scipy/sparse/_compressed.py:1077: ValueError

related issue: scipy/scipy#8678

To Reproduce
To help us reproducing this bug, please provide information below:

  1. Your Python version
  2. The version of Mars you use
  3. Versions of crucial packages, such as numpy, scipy and pandas
  4. Full stack of the error.
  5. Minimized code to reproduce the error.

Expected behavior
A clear and concise description of what you expected to happen.

Additional context
Add any other context about the problem here.