Joblib Unable to unpickle class properly

I am getting Attribute error while trying to unpickle my object using joblib. Should I separately pickle my classes and then unpickle them.

class LengthTransformer(TransformerMixin):

    def transform(self, X, **transform_params):
        length = pd.DataFrame(X.apply(lambda x: len(x)))
        return length

    def fit(self, X, y=None, **fit_params):
        return self

pipeline = Pipeline([('features', FeatureUnion([
                                 ('ngram_tf_idf', Pipeline([
                                     ('vect', CountVectorizer()),
                                     ('tfidf', TfidfTransformer())
                                 ])),
                                 ('length',LengthTransformer())
                            ])),
                             ('clf', clf),])

vectorizer_name = 'fpath'+name

joblib.dump(pipeline.steps[0][1],vectorizer_name+'_vect_'+str(int(accuracy*100)) +'.pkl')

Loading the pickled files here

vect_file = "MultinomialNB_vect_80.pkl"
vect = joblib.load(vect_file)

I get the following error. I believe it is bcuz it doesn`t pickle my classes. I read the pickle can be used to serialize the classes but I am using joblib for better performance.

 AttributeError                            Traceback (most recent call last)
<ipython-input-4-a5d89ba85002> in <module>()
      4 vect_file = "MultinomialNB_vect_80.pkl"
----> 5 vect_n = joblib.load(vect_file)
      6 print(vect_n)
      7 # tfid = vect_n.steps[1][1]

~/anaconda2/envs/py35/lib/python3.5/site-packages/sklearn/externals/joblib/numpy_pickle.py in load(filename, mmap_mode)
    576                     return load_compatibility(fobj)
    577 
--> 578                 obj = _unpickle(fobj, filename, mmap_mode)
    579 
    580     return obj

~/anaconda2/envs/py35/lib/python3.5/site-packages/sklearn/externals/joblib/numpy_pickle.py in _unpickle(fobj, filename, mmap_mode)
    506     obj = None
    507     try:
--> 508         obj = unpickler.load()
    509         if unpickler.compat_mode:
    510             warnings.warn("The file '%s' has been generated with a "

~/anaconda2/envs/py35/lib/python3.5/pickle.py in load(self)
   1041                     raise EOFError
   1042                 assert isinstance(key, bytes_types)
-> 1043                 dispatch[key[0]](self)
   1044         except _Stop as stopinst:
   1045             return stopinst.value

~/anaconda2/envs/py35/lib/python3.5/pickle.py in load_global(self)
   1340         module = self.readline()[:-1].decode("utf-8")
   1341         name = self.readline()[:-1].decode("utf-8")
-> 1342         klass = self.find_class(module, name)
   1343         self.append(klass)
   1344     dispatch[GLOBAL[0]] = load_global

~/anaconda2/envs/py35/lib/python3.5/pickle.py in find_class(self, module, name)
   1394             return _getattribute(sys.modules[module], name)[0]
   1395         else:
-> 1396             return getattr(sys.modules[module], name)
   1397 
   1398     def load_reduce(self):

AttributeError: module '__main__' has no attribute 'LengthTransformer'

---

在一个单独的模块中定义你的类，并将该模块导入到执行pickle dump的程序和执行pickle加载的程序中。

链接地址: http://www.djcxy.com/p/64834.html

上一篇: 代码审查协作

下一篇: Joblib无法正确拔除类