喵十八の小窝
0%
1.7k 字 10 分钟

FATE学习：运行过程中产生的数据（四） model_cache

Posted on In

综述

存储fate 训练过程中的模型
目录: /data/projects/fate/model_local_cache/{model_version}/{model_id}

内容

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
(app-root) bash-4.2# ls -R
.:
define  pipeline.pb  variables

./define:
define_meta.yaml  proto

./define/proto:
boosting-tree-model-meta.proto   data-io-param.proto          feature-selection-meta.proto   hetero-kmeans-param.proto    lr-model-meta.proto   onehot-param.proto         poisson-model-meta.proto   sir-param.proto
boosting-tree-model-param.proto  feature-binning-meta.proto   feature-selection-param.proto  hetero-nn-model-meta.proto   lr-model-param.proto  one-vs-rest-param.proto    poisson-model-param.proto  statistic-meta.proto
column-expand-meta.proto         feature-binning-param.proto  ftl-model-meta.proto           hetero-nn-model-param.proto  nn-model-meta.proto   pearson-model-meta.proto   psi-model-meta.proto       statistic-param.proto
column-expand-param.proto        feature-scale-meta.proto     ftl-model-param.proto          linr-model-meta.proto        nn-model-param.proto  pearson-model-param.proto  psi-model-param.proto
data-io-meta.proto               feature-scale-param.proto    hetero-kmeans-meta.proto       linr-model-param.proto       onehot-meta.proto     pipeline.proto             sir-meta.proto

./variables:
data  index

./variables/data:
pipeline

./variables/data/pipeline:
pipeline

./variables/data/pipeline/pipeline:
Pipeline

./variables/index:

其中重要的内容为
define_meta.yaml 模型的元数据定义
如upload 为

1
2
3
4
5
6
7
8
describe: This is the model definition meta
component_define:
  pipeline:
    module_name: Pipeline
model_proto:
  pipeline:
    pipeline:
      Pipeline: Pipeline

lr的为
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
describe: This is the model definition meta
component_define:
  dataio_0:
    module_name: DataIO
  hetero_lr_0:
    module_name: HeteroLR
  pipeline:
    module_name: Pipeline
model_proto:
  dataio_0:
    dataio:
      DataIOMeta: DataIOMeta
      DataIOParam: DataIOParam
    default:
      DataIOMeta: DataIOMeta
      DataIOParam: DataIOParam
  hetero_lr_0:
    hetero_lr:
      HeteroLogisticRegressionMeta: LRModelMeta
      HeteroLogisticRegressionParam: LRModelParam
    default:
      HeteroLogisticRegressionMeta: LRModelMeta
      HeteroLogisticRegressionParam: LRModelParam
  pipeline:
    pipeline:
      Pipeline: Pipeline

/define/proto 目录下，为各个组件的Protobuf

pipeline.pb 类似Run Time Conf，定义了各项参数

1
{"components": {"dataio_0": {"module": "DataIO", "input": {"model": ["pipeline.dataio_0.dataio"], "data": {"data": ["args.eval_data"]}}, "output": {"data": ["train"]}, "CodePath": "federatedml/util/data_io.py/DataIO"}, "intersection_0": {"module": "Intersection", "output": {"data": ["train"]}, "input": {"data": {"data": ["dataio_0.train"]}}, "CodePath": "federatedml/statistic/intersect/intersect_model.py/IntersectGuest"}, "hetero_lr_0": {"module": "HeteroLR", "input": {"model": ["pipeline.hetero_lr_0.hetero_lr"], "data": {"test_data": ["intersection_0.train"]}}, "output": {"data": ["train"]}, "CodePath": "federatedml/linear_model/logistic_regression/hetero_logistic_regression/hetero_lr_guest.py/HeteroLRGuest"}}}�{"components": {"dataio_0": {"module": "DataIO", "input": {"data": {"data": ["args.train_data"]}}, "output": {"data": ["train"], "model": ["dataio"]}}, "intersection_0": {"module": "Intersection", "input": {"data": {"data": ["dataio_0.train"]}}, "output": {"data": ["train"]}}, "hetero_lr_0": {"module": "HeteroLR", "input": {"data": {"train_data": ["intersection_0.train"]}}, "output": {"data": ["train"], "model": ["hetero_lr"]}}}}�{"initiator": {"role": "guest", "party_id": 9989}, "job_parameters": {"job_type": "train", "work_mode": 1, "backend": 0, "computing_engine": "EGGROLL", "federation_engine": "EGGROLL", "storage_engine": "EGGROLL", "engines_address": {}, "federated_mode": "MULTIPLE", "task_parallelism": 1, "computing_partitions": 4, "federated_status_collect_type": "PULL", "model_id": "arbiter-10000#guest-9989#host-10000#model", "model_version": "20210907124033978481324", "eggroll_run": {}, "spark_run": {}, "rabbitmq_run": {}, "adaptation_parameters": {"task_nodes": 1, "task_cores_per_node": 4, "task_memory_per_node": 0, "request_task_cores": 4, "if_initiator_baseline": true}}, "role": {"guest": [9989], "host": [10000], "arbiter": [10000]}, "role_parameters": {"guest": {"args": {"data": {"train_data": [{"name": "agame11512_20210906", "namespace": "cl"}], "eval_data": [{"name": "agame11512_20210906", "namespace": "cl"}]}}, "dataio_0": {"with_label": [true], "label_name": ["y"], "label_type": ["int"], "output_format": ["dense"], "missing_fill": [true], "outlier_replace": [true]}, "evaluation_0": {"eval_type": ["binary"], "pos_label": [1]}}, "host": {"args": {"data": {"train_data": [{"name": "20210906_test_cl_agame11512", "namespace": "experiment"}], "eval_data": [{"name": "20210906_test_cl_agame11512", "namespace": "experiment"}]}}, "dataio_0": {"with_label": [false], "output_format": ["dense"], "outlier_replace": [true]}, "evaluation_0": {"need_run": [false]}}}, "algorithm_parameters": {"hetero_lr_0": {"penalty": "L2", "optimizer": "nesterov_momentum_sgd", "tol": 0.0001, "alpha": 0.01, "max_iter": 1, "early_stop": "weight_diff", "batch_size": -1, "learning_rate": 0.15, "init_param": {"init_method": "random_uniform"}, "sqn_param": {"update_interval_L": 3, "memory_M": 5, "sample_size": 5000, "random_seed": null}, "cv_param": {"n_splits": 1, "shuffle": false, "random_seed": 103, "need_cv": false}}, "intersect_0": {"intersect_method": "rsa", "sync_intersect_ids": true, "only_output_key": false}}, "config": "/data/projects/fate/python/cl/fc/config/20210906/train_20210906_agame11512.json", "dsl": "/data/projects/fate/python/cl/fc/env/pattern/train_job_dsl.json", "function": "submit_job"}"1.5.1*)arbiter-10000#guest-9989#host-10000#model2202109071240339784813248J6{"guest": [9989], "host": [10000], "arbiter": [10000]}PZguest`�Nj�{"initiator": {"role": "guest", "party_id": 9989}, "job_parameters": {"job_type": "train", "work_mode": 1, "backend": 0, "computing_engine": "EGGROLL", "federation_engine": "EGGROLL", "storage_engine": "EGGROLL", "engines_address": {"computing": {"cores_per_node": 20, "nodes": 1}, "federation": {"host": "rollsite", "port": 9370}, "storage": {"cores_per_node": 20, "nodes": 1}}, "federated_mode": "MULTIPLE", "task_parallelism": 1, "computing_partitions": 4, "federated_status_collect_type": "PULL", "model_id": "arbiter-10000#guest-9989#host-10000#model", "model_version": "20210907124033978481324", "eggroll_run": {"eggroll.session.processors.per.node": 4}, "spark_run": {}, "rabbitmq_run": {}, "adaptation_parameters": {"task_nodes": 1, "task_cores_per_node": 4, "task_memory_per_node": 0, "request_task_cores": 4, "if_initiator_baseline": false}}, "role": {"guest": [9989], "host": [10000], "arbiter": [10000]}, "role_parameters": {"guest": {"args": {"data": {"train_data": [{"name": "agame11512_20210906", "namespace": "cl"}], "eval_data": [{"name": "agame11512_20210906", "namespace": "cl"}]}}, "dataio_0": {"with_label": [true], "label_name": ["y"], "label_type": ["int"], "output_format": ["dense"], "missing_fill": [true], "outlier_replace": [true]}, "evaluation_0": {"eval_type": ["binary"], "pos_label": [1]}}, "host": {"args": {"data": {"train_data": [{"name": "20210906_test_cl_agame11512", "namespace": "experiment"}], "eval_data": [{"name": "20210906_test_cl_agame11512", "namespace": "experiment"}]}}, "dataio_0": {"with_label": [false], "output_format": ["dense"], "outlier_replace": [true]}, "evaluation_0": {"need_run": [false]}}}, "algorithm_parameters": {"hetero_lr_0": {"penalty": "L2", "optimizer": "nesterov_momentum_sgd", "tol": 0.0001, "alpha": 0.01, "max_iter": 1, "early_stop": "weight_diff", "batch_size": -1, "learning_rate": 0.15, "init_param": {"init_method": "random_uniform"}, "sqn_param": {"update_interval_L": 3, "memory_M": 5, "sample_size": 5000, "random_seed": null}, "cv_param": {"n_splits": 1, "shuffle": false, "random_seed": 103, "need_cv": false}}, "intersect_0": {"intersect_method": "rsa", "sync_intersect_ids": true, "only_output_key": false}}, "config": "/data/projects/fate/python/cl/fc/config/20210906/train_20210906_agame11512.json", "dsl": "/data/projects/fate/python/cl/fc/env/pattern/train_job_dsl.json", "function": "submit_job"}r{}

/variables/data
记录当前part 的模型的参数