ดูบน TensorFlow.org | ทำงานใน Google Colab | ดูบน GitHub | ดาวน์โหลดโน๊ตบุ๊ค | ดูรุ่น TF Hub |
ในการกวดวิชานี้คุณจะได้เรียนรู้วิธีการใช้ ความเป็นธรรมตัวชี้วัด ในการประเมิน embeddings จาก TF Hub โน๊ตบุ๊คนี้ใช้ ความคิดเห็นชุดพลเรือน
ติดตั้ง
ติดตั้งไลบรารีที่จำเป็น
!pip install -q -U pip==20.2
!pip install fairness-indicators \
"absl-py==0.12.0" \
"pyarrow==2.0.0" \
"apache-beam==2.34.0" \
"avro-python3==1.9.1"
นำเข้าไลบรารีที่จำเป็นอื่น ๆ
import os
import tempfile
import apache_beam as beam
from datetime import datetime
import tensorflow as tf
import tensorflow_hub as hub
import tensorflow_model_analysis as tfma
from tensorflow_model_analysis.addons.fairness.view import widget_view
from tensorflow_model_analysis.addons.fairness.post_export_metrics import fairness_indicators
from fairness_indicators import example_model
from fairness_indicators.tutorial_utils import util
ERROR: Traceback (most recent call last): File "/tmpfs/src/tf_docs_env/lib/python3.7/site-packages/apache_beam/io/gcp/bigquery.py", line 341, in <module> import google.cloud.bigquery_storage_v1 as bq_storage ModuleNotFoundError: No module named 'google.cloud.bigquery_storage_v1'
ชุดข้อมูล
ในสมุดบันทึกนี้คุณทำงานร่วมกับ พลเรือนชุดคอมเมนต์ ซึ่งมีประมาณ 2 ล้านความคิดเห็นของประชาชนทำให้ประชาชนโดย พลเรือนคอมเมนต์แพลตฟอร์ม ในปี 2017 สำหรับการวิจัยอย่างต่อเนื่อง ความพยายามนี้ได้รับการสนับสนุนโดย Jigsaw ซึ่งเป็นเจ้าภาพการแข่งขัน Kaggle เพื่อช่วยจำแนกความคิดเห็นที่เป็นพิษรวมทั้งลดความลำเอียงของโมเดลโดยไม่ได้ตั้งใจ
ความคิดเห็นที่เป็นข้อความแต่ละรายการในชุดข้อมูลมีป้ายกำกับความเป็นพิษ โดยป้ายกำกับจะเป็น 1 หากความคิดเห็นเป็นพิษ และ 0 หากความคิดเห็นไม่เป็นพิษ ภายในข้อมูล ชุดย่อยของความคิดเห็นจะมีป้ายกำกับด้วยแอตทริบิวต์ข้อมูลประจำตัวที่หลากหลาย รวมถึงหมวดหมู่สำหรับเพศ รสนิยมทางเพศ ศาสนา และเชื้อชาติหรือชาติพันธุ์
เตรียมข้อมูล
TensorFlow แยกวิเคราะห์คุณสมบัติจากข้อมูลโดยใช้ tf.io.FixedLenFeature
และ tf.io.VarLenFeature
กำหนดคุณลักษณะอินพุต คุณลักษณะเอาต์พุต และคุณลักษณะการแบ่งส่วนอื่นๆ ทั้งหมดที่น่าสนใจ
BASE_DIR = tempfile.gettempdir()
# The input and output features of the classifier
TEXT_FEATURE = 'comment_text'
LABEL = 'toxicity'
FEATURE_MAP = {
# input and output features
LABEL: tf.io.FixedLenFeature([], tf.float32),
TEXT_FEATURE: tf.io.FixedLenFeature([], tf.string),
# slicing features
'sexual_orientation': tf.io.VarLenFeature(tf.string),
'gender': tf.io.VarLenFeature(tf.string),
'religion': tf.io.VarLenFeature(tf.string),
'race': tf.io.VarLenFeature(tf.string),
'disability': tf.io.VarLenFeature(tf.string)
}
IDENTITY_TERMS = ['gender', 'sexual_orientation', 'race', 'religion', 'disability']
ตามค่าเริ่มต้น โน้ตบุ๊กจะดาวน์โหลดชุดข้อมูลเวอร์ชันที่ประมวลผลล่วงหน้า แต่คุณอาจใช้ชุดข้อมูลเดิมและเรียกใช้ขั้นตอนการประมวลผลอีกครั้งหากต้องการ
ในชุดข้อมูลดั้งเดิม ความคิดเห็นแต่ละรายการจะมีป้ายกำกับเป็นเปอร์เซ็นต์ของผู้ประเมินที่เชื่อว่าความคิดเห็นสอดคล้องกับข้อมูลเฉพาะตัว ยกตัวอย่างเช่นอาจจะมีการแสดงความคิดเห็นที่มีป้ายกำกับต่อไปนี้: { male: 0.3, female: 1.0, transgender: 0.0, heterosexual: 0.8, homosexual_gay_or_lesbian: 1.0 }
ขั้นตอนการประมวลผลจะจัดกลุ่มข้อมูลประจำตัวตามหมวดหมู่ (เพศ รสนิยมทางเพศ เป็นต้น) และลบข้อมูลประจำตัวที่มีคะแนนน้อยกว่า 0.5 ดังนั้น ตัวอย่างข้างต้นจะถูกแปลงเป็นดังต่อไปนี้: ของผู้ประเมินที่เชื่อว่าความคิดเห็นสอดคล้องกับเอกลักษณ์เฉพาะ ยกตัวอย่างเช่นการแสดงความคิดเห็นดังกล่าวข้างต้นจะมีป้ายกำกับต่อไปนี้: { gender: [female], sexual_orientation: [heterosexual, homosexual_gay_or_lesbian] }
ดาวน์โหลดชุดข้อมูล
download_original_data = False
if download_original_data:
train_tf_file = tf.keras.utils.get_file('train_tf.tfrecord',
'https://storage.googleapis.com/civil_comments_dataset/train_tf.tfrecord')
validate_tf_file = tf.keras.utils.get_file('validate_tf.tfrecord',
'https://storage.googleapis.com/civil_comments_dataset/validate_tf.tfrecord')
# The identity terms list will be grouped together by their categories
# (see 'IDENTITY_COLUMNS') on threshold 0.5. Only the identity term column,
# text column and label column will be kept after processing.
train_tf_file = util.convert_comments_data(train_tf_file)
validate_tf_file = util.convert_comments_data(validate_tf_file)
else:
train_tf_file = tf.keras.utils.get_file('train_tf_processed.tfrecord',
'https://storage.googleapis.com/civil_comments_dataset/train_tf_processed.tfrecord')
validate_tf_file = tf.keras.utils.get_file('validate_tf_processed.tfrecord',
'https://storage.googleapis.com/civil_comments_dataset/validate_tf_processed.tfrecord')
Downloading data from https://storage.googleapis.com/civil_comments_dataset/train_tf_processed.tfrecord 488161280/488153424 [==============================] - 2s 0us/step 488169472/488153424 [==============================] - 2s 0us/step Downloading data from https://storage.googleapis.com/civil_comments_dataset/validate_tf_processed.tfrecord 324943872/324941336 [==============================] - 9s 0us/step 324952064/324941336 [==============================] - 9s 0us/step
สร้างไปป์ไลน์การวิเคราะห์แบบจำลอง TensorFlow
ห้องสมุดความเป็นธรรมชี้วัดการดำเนินงานใน TensorFlow รุ่นวิเคราะห์ (Tfma) รุ่น โมเดล TFMA รวมโมเดล TensorFlow ด้วยฟังก์ชันเพิ่มเติมเพื่อประเมินและแสดงผลลัพธ์เป็นภาพ การประเมินผลที่เกิดขึ้นจริงที่เกิดขึ้นภายในของ ท่อ Apache Beam
ขั้นตอนที่คุณทำตามเพื่อสร้างไปป์ไลน์ TFMA คือ:
- สร้างโมเดล TensorFlow
- สร้างโมเดล TFMA ที่ด้านบนของโมเดล TensorFlow
- เรียกใช้การวิเคราะห์แบบจำลองในออร์เคสตรา โมเดลตัวอย่างในโน้ตบุ๊กนี้ใช้ Apache Beam เป็นผู้ประสานงาน
def embedding_fairness_result(embedding, identity_term='gender'):
model_dir = os.path.join(BASE_DIR, 'train',
datetime.now().strftime('%Y%m%d-%H%M%S'))
print("Training classifier for " + embedding)
classifier = example_model.train_model(model_dir,
train_tf_file,
LABEL,
TEXT_FEATURE,
FEATURE_MAP,
embedding)
# Create a unique path to store the results for this embedding.
embedding_name = embedding.split('/')[-2]
eval_result_path = os.path.join(BASE_DIR, 'eval_result', embedding_name)
example_model.evaluate_model(classifier,
validate_tf_file,
eval_result_path,
identity_term,
LABEL,
FEATURE_MAP)
return tfma.load_eval_result(output_path=eval_result_path)
เรียกใช้ TFMA & ตัวชี้วัดความเป็นธรรม
ตัวชี้วัดความเป็นธรรม ตัวชี้วัด
ตัวชี้วัดบางส่วนที่ใช้ได้กับตัวชี้วัดความเป็นธรรม ได้แก่
- อัตราติดลบ อัตราลบเท็จ (FNR) และอัตราการติดลบจริง (TNR)
- อัตราบวก อัตราบวกเท็จ (FPR) และอัตราผลบวกจริง (TPR)
- ความแม่นยำ
- ความแม่นยำและการเรียกคืน
- แม่นยำ-เรียกคืนAUC
- ROC AUC
การฝังข้อความ
TF-Hub ให้ embeddings ข้อความหลาย การฝังเหล่านี้จะทำหน้าที่เป็นคอลัมน์คุณลักษณะสำหรับรุ่นต่างๆ บทช่วยสอนนี้ใช้การฝังต่อไปนี้:
- สุ่ม nnlm-en-dim128 : embeddings ข้อความสุ่มนี้ทำหน้าที่เป็นพื้นฐานที่สะดวก
- nnlm-en-dim128 : ข้อความที่ฝังอยู่บนพื้นฐานของ ประสาทน่าจะเป็นภาษารุ่น
- สากลประโยคเข้ารหัส : ข้อความที่ฝังอยู่บนพื้นฐานของ ยูนิเวอร์แซประโยค Encoder
ผลตัวชี้วัดความเป็นธรรม
ตัวชี้วัดความเป็นธรรม Compute กับ embedding_fairness_result
ท่อแล้วทำให้ผลลัพธ์ในความเป็นธรรมดัชนี UI เครื่องมือที่มี widget_view.render_fairness_indicator
สำหรับทุก embeddings ดังกล่าวข้างต้น
สุ่ม NNLM
eval_result_random_nnlm = embedding_fairness_result('https://tfhub.dev/google/random-nnlm-en-dim128/1')
Training classifier for https://tfhub.dev/google/random-nnlm-en-dim128/1 INFO:tensorflow:Using default config. INFO:tensorflow:Using default config. INFO:tensorflow:Using config: {'_model_dir': '/tmp/train/20220107-182244', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true graph_options { rewrite_options { meta_optimizer_iterations: ONE } } , '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1} INFO:tensorflow:Using config: {'_model_dir': '/tmp/train/20220107-182244', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true graph_options { rewrite_options { meta_optimizer_iterations: ONE } } , '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1} WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow/python/training/training_util.py:397: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version. Instructions for updating: Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts. WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow/python/training/training_util.py:397: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version. Instructions for updating: Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts. INFO:tensorflow:Calling model_fn. INFO:tensorflow:Calling model_fn. INFO:tensorflow:Saver not created because there are no variables in the graph to restore 2022-01-07 18:22:54.196242: W tensorflow/core/common_runtime/graph_constructor.cc:1511] Importing a graph with a lower producer version 26 into an existing graph with producer version 987. Shape inference will have run different parts of the graph with different producer versions. INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Saver not created because there are no variables in the graph to restore WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow_estimator/python/estimator/canned/head.py:400: NumericColumn._get_dense_tensor (from tensorflow.python.feature_column.feature_column_v2) is deprecated and will be removed in a future version. Instructions for updating: The old _FeatureColumn APIs are being deprecated. Please use the new FeatureColumn APIs instead. WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow_estimator/python/estimator/canned/head.py:400: NumericColumn._get_dense_tensor (from tensorflow.python.feature_column.feature_column_v2) is deprecated and will be removed in a future version. Instructions for updating: The old _FeatureColumn APIs are being deprecated. Please use the new FeatureColumn APIs instead. WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow/python/feature_column/feature_column.py:2188: NumericColumn._transform_feature (from tensorflow.python.feature_column.feature_column_v2) is deprecated and will be removed in a future version. Instructions for updating: The old _FeatureColumn APIs are being deprecated. Please use the new FeatureColumn APIs instead. WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow/python/feature_column/feature_column.py:2188: NumericColumn._transform_feature (from tensorflow.python.feature_column.feature_column_v2) is deprecated and will be removed in a future version. Instructions for updating: The old _FeatureColumn APIs are being deprecated. Please use the new FeatureColumn APIs instead. WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow/python/training/adagrad.py:139: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version. Instructions for updating: Call initializer instance with the dtype argument instead of passing it to the constructor WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow/python/training/adagrad.py:139: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version. Instructions for updating: Call initializer instance with the dtype argument instead of passing it to the constructor INFO:tensorflow:Done calling model_fn. INFO:tensorflow:Done calling model_fn. INFO:tensorflow:Create CheckpointSaverHook. INFO:tensorflow:Create CheckpointSaverHook. INFO:tensorflow:Graph was finalized. INFO:tensorflow:Graph was finalized. INFO:tensorflow:Running local_init_op. INFO:tensorflow:Running local_init_op. INFO:tensorflow:Done running local_init_op. INFO:tensorflow:Done running local_init_op. INFO:tensorflow:Calling checkpoint listeners before saving checkpoint 0... INFO:tensorflow:Calling checkpoint listeners before saving checkpoint 0... INFO:tensorflow:Saving checkpoints for 0 into /tmp/train/20220107-182244/model.ckpt. INFO:tensorflow:Saving checkpoints for 0 into /tmp/train/20220107-182244/model.ckpt. INFO:tensorflow:Calling checkpoint listeners after saving checkpoint 0... INFO:tensorflow:Calling checkpoint listeners after saving checkpoint 0... INFO:tensorflow:loss = 60.23522, step = 0 INFO:tensorflow:loss = 60.23522, step = 0 INFO:tensorflow:global_step/sec: 78.2958 INFO:tensorflow:global_step/sec: 78.2958 INFO:tensorflow:loss = 67.36491, step = 100 (1.279 sec) INFO:tensorflow:loss = 67.36491, step = 100 (1.279 sec) INFO:tensorflow:global_step/sec: 85.8245 INFO:tensorflow:global_step/sec: 85.8245 INFO:tensorflow:loss = 57.875557, step = 200 (1.165 sec) INFO:tensorflow:loss = 57.875557, step = 200 (1.165 sec) INFO:tensorflow:global_step/sec: 83.7495 INFO:tensorflow:global_step/sec: 83.7495 INFO:tensorflow:loss = 61.091763, step = 300 (1.194 sec) INFO:tensorflow:loss = 61.091763, step = 300 (1.194 sec) INFO:tensorflow:global_step/sec: 83.0013 INFO:tensorflow:global_step/sec: 83.0013 INFO:tensorflow:loss = 62.251183, step = 400 (1.205 sec) INFO:tensorflow:loss = 62.251183, step = 400 (1.205 sec) INFO:tensorflow:global_step/sec: 83.4782 INFO:tensorflow:global_step/sec: 83.4782 INFO:tensorflow:loss = 56.21132, step = 500 (1.198 sec) INFO:tensorflow:loss = 56.21132, step = 500 (1.198 sec) INFO:tensorflow:global_step/sec: 87.0099 INFO:tensorflow:global_step/sec: 87.0099 INFO:tensorflow:loss = 57.211937, step = 600 (1.149 sec) INFO:tensorflow:loss = 57.211937, step = 600 (1.149 sec) INFO:tensorflow:global_step/sec: 86.7988 INFO:tensorflow:global_step/sec: 86.7988 INFO:tensorflow:loss = 62.16255, step = 700 (1.152 sec) INFO:tensorflow:loss = 62.16255, step = 700 (1.152 sec) INFO:tensorflow:global_step/sec: 88.1099 INFO:tensorflow:global_step/sec: 88.1099 INFO:tensorflow:loss = 58.081688, step = 800 (1.135 sec) INFO:tensorflow:loss = 58.081688, step = 800 (1.135 sec) INFO:tensorflow:global_step/sec: 85.3134 INFO:tensorflow:global_step/sec: 85.3134 INFO:tensorflow:loss = 57.763985, step = 900 (1.172 sec) INFO:tensorflow:loss = 57.763985, step = 900 (1.172 sec) INFO:tensorflow:Calling checkpoint listeners before saving checkpoint 1000... INFO:tensorflow:Calling checkpoint listeners before saving checkpoint 1000... INFO:tensorflow:Saving checkpoints for 1000 into /tmp/train/20220107-182244/model.ckpt. INFO:tensorflow:Saving checkpoints for 1000 into /tmp/train/20220107-182244/model.ckpt. INFO:tensorflow:Calling checkpoint listeners after saving checkpoint 1000... INFO:tensorflow:Calling checkpoint listeners after saving checkpoint 1000... INFO:tensorflow:Loss for final step: 59.963802. INFO:tensorflow:Loss for final step: 59.963802. WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow_model_analysis/eval_saved_model/encoding.py:132: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version. Instructions for updating: This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info. WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow_model_analysis/eval_saved_model/encoding.py:132: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version. Instructions for updating: This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info. INFO:tensorflow:Calling model_fn. INFO:tensorflow:Calling model_fn. INFO:tensorflow:Saver not created because there are no variables in the graph to restore 2022-01-07 18:23:11.033169: W tensorflow/core/common_runtime/graph_constructor.cc:1511] Importing a graph with a lower producer version 26 into an existing graph with producer version 987. Shape inference will have run different parts of the graph with different producer versions. INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Saver not created because there are no variables in the graph to restore WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow_estimator/python/estimator/canned/head.py:640: auc (from tensorflow.python.ops.metrics_impl) is deprecated and will be removed in a future version. Instructions for updating: The value of AUC returned by this may race with the update so this is deprecated. Please use tf.keras.metrics.AUC instead. WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow_estimator/python/estimator/canned/head.py:640: auc (from tensorflow.python.ops.metrics_impl) is deprecated and will be removed in a future version. Instructions for updating: The value of AUC returned by this may race with the update so this is deprecated. Please use tf.keras.metrics.AUC instead. WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead. WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead. WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead. WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead. INFO:tensorflow:Done calling model_fn. INFO:tensorflow:Done calling model_fn. INFO:tensorflow:Signatures INCLUDED in export for Classify: None INFO:tensorflow:Signatures INCLUDED in export for Classify: None INFO:tensorflow:Signatures INCLUDED in export for Regress: None INFO:tensorflow:Signatures INCLUDED in export for Regress: None INFO:tensorflow:Signatures INCLUDED in export for Predict: None INFO:tensorflow:Signatures INCLUDED in export for Predict: None INFO:tensorflow:Signatures INCLUDED in export for Train: None INFO:tensorflow:Signatures INCLUDED in export for Train: None INFO:tensorflow:Signatures INCLUDED in export for Eval: ['eval'] INFO:tensorflow:Signatures INCLUDED in export for Eval: ['eval'] WARNING:tensorflow:Export includes no default signature! WARNING:tensorflow:Export includes no default signature! INFO:tensorflow:Restoring parameters from /tmp/train/20220107-182244/model.ckpt-1000 INFO:tensorflow:Restoring parameters from /tmp/train/20220107-182244/model.ckpt-1000 INFO:tensorflow:Assets added to graph. INFO:tensorflow:Assets added to graph. INFO:tensorflow:Assets written to: /tmp/tfma_eval_model/temp-1641579790/assets INFO:tensorflow:Assets written to: /tmp/tfma_eval_model/temp-1641579790/assets INFO:tensorflow:SavedModel written to: /tmp/tfma_eval_model/temp-1641579790/saved_model.pb INFO:tensorflow:SavedModel written to: /tmp/tfma_eval_model/temp-1641579790/saved_model.pb WARNING:absl:Tensorflow version (2.8.0-rc0) found. Note that TFMA support for TF 2.0 is currently in beta WARNING:apache_beam.runners.interactive.interactive_environment:Dependencies required for Interactive Beam PCollection visualization are not available, please use: `pip install apache-beam[interactive]` to install necessary dependencies to enable all data visualization features. WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter. WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow_model_analysis/eval_saved_model/load.py:164: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version. Instructions for updating: This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0. WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow_model_analysis/eval_saved_model/load.py:164: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version. Instructions for updating: This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0. INFO:tensorflow:Restoring parameters from /tmp/tfma_eval_model/1641579790/variables/variables INFO:tensorflow:Restoring parameters from /tmp/tfma_eval_model/1641579790/variables/variables WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow_model_analysis/eval_saved_model/graph_ref.py:184: get_tensor_from_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version. Instructions for updating: This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.get_tensor_from_tensor_info or tf.compat.v1.saved_model.get_tensor_from_tensor_info. WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow_model_analysis/eval_saved_model/graph_ref.py:184: get_tensor_from_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version. Instructions for updating: This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.get_tensor_from_tensor_info or tf.compat.v1.saved_model.get_tensor_from_tensor_info. WARNING:apache_beam.io.tfrecordio:Couldn't find python-snappy so the implementation of _TFRecordUtil._masked_crc32c is not as fast as it could be. WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow_model_analysis/writers/metrics_plots_and_validations_writer.py:107: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version. Instructions for updating: Use eager execution and: `tf.data.TFRecordDataset(path)` WARNING:tensorflow:From /tmpfs/src/tf_docs_env/lib/python3.7/site-packages/tensorflow_model_analysis/writers/metrics_plots_and_validations_writer.py:107: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version. Instructions for updating: Use eager execution and: `tf.data.TFRecordDataset(path)`
widget_view.render_fairness_indicator(eval_result=eval_result_random_nnlm)
FairnessIndicatorViewer(slicingMetrics=[{'sliceValue': 'Overall', 'slice': 'Overall', 'metrics': {'post_export…
NNLM
eval_result_nnlm = embedding_fairness_result('https://tfhub.dev/google/nnlm-en-dim128/1')
Training classifier for https://tfhub.dev/google/nnlm-en-dim128/1 INFO:tensorflow:Using default config. INFO:tensorflow:Using default config. INFO:tensorflow:Using config: {'_model_dir': '/tmp/train/20220107-182524', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true graph_options { rewrite_options { meta_optimizer_iterations: ONE } } , '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1} INFO:tensorflow:Using config: {'_model_dir': '/tmp/train/20220107-182524', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true graph_options { rewrite_options { meta_optimizer_iterations: ONE } } , '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1} INFO:tensorflow:Calling model_fn. INFO:tensorflow:Calling model_fn. INFO:tensorflow:Saver not created because there are no variables in the graph to restore 2022-01-07 18:25:24.785154: W tensorflow/core/common_runtime/graph_constructor.cc:1511] Importing a graph with a lower producer version 26 into an existing graph with producer version 987. Shape inference will have run different parts of the graph with different producer versions. INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Done calling model_fn. INFO:tensorflow:Done calling model_fn. INFO:tensorflow:Create CheckpointSaverHook. INFO:tensorflow:Create CheckpointSaverHook. INFO:tensorflow:Graph was finalized. INFO:tensorflow:Graph was finalized. INFO:tensorflow:Running local_init_op. INFO:tensorflow:Running local_init_op. INFO:tensorflow:Done running local_init_op. INFO:tensorflow:Done running local_init_op. INFO:tensorflow:Calling checkpoint listeners before saving checkpoint 0... INFO:tensorflow:Calling checkpoint listeners before saving checkpoint 0... INFO:tensorflow:Saving checkpoints for 0 into /tmp/train/20220107-182524/model.ckpt. INFO:tensorflow:Saving checkpoints for 0 into /tmp/train/20220107-182524/model.ckpt. INFO:tensorflow:Calling checkpoint listeners after saving checkpoint 0... INFO:tensorflow:Calling checkpoint listeners after saving checkpoint 0... INFO:tensorflow:loss = 58.637047, step = 0 INFO:tensorflow:loss = 58.637047, step = 0 INFO:tensorflow:global_step/sec: 75.6907 INFO:tensorflow:global_step/sec: 75.6907 INFO:tensorflow:loss = 56.208035, step = 100 (1.323 sec) INFO:tensorflow:loss = 56.208035, step = 100 (1.323 sec) INFO:tensorflow:global_step/sec: 85.4193 INFO:tensorflow:global_step/sec: 85.4193 INFO:tensorflow:loss = 47.563675, step = 200 (1.170 sec) INFO:tensorflow:loss = 47.563675, step = 200 (1.170 sec) INFO:tensorflow:global_step/sec: 85.3916 INFO:tensorflow:global_step/sec: 85.3916 INFO:tensorflow:loss = 56.227097, step = 300 (1.171 sec) INFO:tensorflow:loss = 56.227097, step = 300 (1.171 sec) INFO:tensorflow:global_step/sec: 85.7359 INFO:tensorflow:global_step/sec: 85.7359 INFO:tensorflow:loss = 55.668434, step = 400 (1.166 sec) INFO:tensorflow:loss = 55.668434, step = 400 (1.166 sec) INFO:tensorflow:global_step/sec: 85.6231 INFO:tensorflow:global_step/sec: 85.6231 INFO:tensorflow:loss = 41.7245, step = 500 (1.168 sec) INFO:tensorflow:loss = 41.7245, step = 500 (1.168 sec) INFO:tensorflow:global_step/sec: 85.1399 INFO:tensorflow:global_step/sec: 85.1399 INFO:tensorflow:loss = 45.596313, step = 600 (1.174 sec) INFO:tensorflow:loss = 45.596313, step = 600 (1.174 sec) INFO:tensorflow:global_step/sec: 83.6346 INFO:tensorflow:global_step/sec: 83.6346 INFO:tensorflow:loss = 51.108143, step = 700 (1.196 sec) INFO:tensorflow:loss = 51.108143, step = 700 (1.196 sec) INFO:tensorflow:global_step/sec: 85.4834 INFO:tensorflow:global_step/sec: 85.4834 INFO:tensorflow:loss = 47.63583, step = 800 (1.170 sec) INFO:tensorflow:loss = 47.63583, step = 800 (1.170 sec) INFO:tensorflow:global_step/sec: 86.7353 INFO:tensorflow:global_step/sec: 86.7353 INFO:tensorflow:loss = 48.044117, step = 900 (1.153 sec) INFO:tensorflow:loss = 48.044117, step = 900 (1.153 sec) INFO:tensorflow:Calling checkpoint listeners before saving checkpoint 1000... INFO:tensorflow:Calling checkpoint listeners before saving checkpoint 1000... INFO:tensorflow:Saving checkpoints for 1000 into /tmp/train/20220107-182524/model.ckpt. INFO:tensorflow:Saving checkpoints for 1000 into /tmp/train/20220107-182524/model.ckpt. INFO:tensorflow:Calling checkpoint listeners after saving checkpoint 1000... INFO:tensorflow:Calling checkpoint listeners after saving checkpoint 1000... INFO:tensorflow:Loss for final step: 50.57175. INFO:tensorflow:Loss for final step: 50.57175. INFO:tensorflow:Calling model_fn. INFO:tensorflow:Calling model_fn. INFO:tensorflow:Saver not created because there are no variables in the graph to restore 2022-01-07 18:25:40.091474: W tensorflow/core/common_runtime/graph_constructor.cc:1511] Importing a graph with a lower producer version 26 into an existing graph with producer version 987. Shape inference will have run different parts of the graph with different producer versions. INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Saver not created because there are no variables in the graph to restore WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead. WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead. WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead. WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead. INFO:tensorflow:Done calling model_fn. INFO:tensorflow:Done calling model_fn. INFO:tensorflow:Signatures INCLUDED in export for Classify: None INFO:tensorflow:Signatures INCLUDED in export for Classify: None INFO:tensorflow:Signatures INCLUDED in export for Regress: None INFO:tensorflow:Signatures INCLUDED in export for Regress: None INFO:tensorflow:Signatures INCLUDED in export for Predict: None INFO:tensorflow:Signatures INCLUDED in export for Predict: None INFO:tensorflow:Signatures INCLUDED in export for Train: None INFO:tensorflow:Signatures INCLUDED in export for Train: None INFO:tensorflow:Signatures INCLUDED in export for Eval: ['eval'] INFO:tensorflow:Signatures INCLUDED in export for Eval: ['eval'] WARNING:tensorflow:Export includes no default signature! WARNING:tensorflow:Export includes no default signature! INFO:tensorflow:Restoring parameters from /tmp/train/20220107-182524/model.ckpt-1000 INFO:tensorflow:Restoring parameters from /tmp/train/20220107-182524/model.ckpt-1000 INFO:tensorflow:Assets added to graph. INFO:tensorflow:Assets added to graph. INFO:tensorflow:Assets written to: /tmp/tfma_eval_model/temp-1641579940/assets INFO:tensorflow:Assets written to: /tmp/tfma_eval_model/temp-1641579940/assets INFO:tensorflow:SavedModel written to: /tmp/tfma_eval_model/temp-1641579940/saved_model.pb INFO:tensorflow:SavedModel written to: /tmp/tfma_eval_model/temp-1641579940/saved_model.pb WARNING:absl:Tensorflow version (2.8.0-rc0) found. Note that TFMA support for TF 2.0 is currently in beta WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter. INFO:tensorflow:Restoring parameters from /tmp/tfma_eval_model/1641579940/variables/variables INFO:tensorflow:Restoring parameters from /tmp/tfma_eval_model/1641579940/variables/variables
widget_view.render_fairness_indicator(eval_result=eval_result_nnlm)
FairnessIndicatorViewer(slicingMetrics=[{'sliceValue': 'Overall', 'slice': 'Overall', 'metrics': {'label/mean'…
ตัวเข้ารหัสประโยคสากล
eval_result_use = embedding_fairness_result('https://tfhub.dev/google/universal-sentence-encoder/2')
Training classifier for https://tfhub.dev/google/universal-sentence-encoder/2 INFO:tensorflow:Using default config. INFO:tensorflow:Using default config. INFO:tensorflow:Using config: {'_model_dir': '/tmp/train/20220107-182759', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true graph_options { rewrite_options { meta_optimizer_iterations: ONE } } , '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1} INFO:tensorflow:Using config: {'_model_dir': '/tmp/train/20220107-182759', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true graph_options { rewrite_options { meta_optimizer_iterations: ONE } } , '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1} INFO:tensorflow:Calling model_fn. INFO:tensorflow:Calling model_fn. 2022-01-07 18:28:15.955057: W tensorflow/core/common_runtime/graph_constructor.cc:1511] Importing a graph with a lower producer version 26 into an existing graph with producer version 987. Shape inference will have run different parts of the graph with different producer versions. INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Done calling model_fn. INFO:tensorflow:Done calling model_fn. INFO:tensorflow:Create CheckpointSaverHook. INFO:tensorflow:Create CheckpointSaverHook. INFO:tensorflow:Graph was finalized. INFO:tensorflow:Graph was finalized. INFO:tensorflow:Running local_init_op. INFO:tensorflow:Running local_init_op. INFO:tensorflow:Done running local_init_op. INFO:tensorflow:Done running local_init_op. INFO:tensorflow:Calling checkpoint listeners before saving checkpoint 0... INFO:tensorflow:Calling checkpoint listeners before saving checkpoint 0... INFO:tensorflow:Saving checkpoints for 0 into /tmp/train/20220107-182759/model.ckpt. INFO:tensorflow:Saving checkpoints for 0 into /tmp/train/20220107-182759/model.ckpt. INFO:tensorflow:Calling checkpoint listeners after saving checkpoint 0... INFO:tensorflow:Calling checkpoint listeners after saving checkpoint 0... INFO:tensorflow:loss = 59.228935, step = 0 INFO:tensorflow:loss = 59.228935, step = 0 INFO:tensorflow:global_step/sec: 8.64079 INFO:tensorflow:global_step/sec: 8.64079 INFO:tensorflow:loss = 50.28162, step = 100 (11.575 sec) INFO:tensorflow:loss = 50.28162, step = 100 (11.575 sec) INFO:tensorflow:global_step/sec: 8.72597 INFO:tensorflow:global_step/sec: 8.72597 INFO:tensorflow:loss = 46.290745, step = 200 (11.460 sec) INFO:tensorflow:loss = 46.290745, step = 200 (11.460 sec) INFO:tensorflow:global_step/sec: 9.02825 INFO:tensorflow:global_step/sec: 9.02825 INFO:tensorflow:loss = 48.490734, step = 300 (11.076 sec) INFO:tensorflow:loss = 48.490734, step = 300 (11.076 sec) INFO:tensorflow:global_step/sec: 9.01342 INFO:tensorflow:global_step/sec: 9.01342 INFO:tensorflow:loss = 44.54372, step = 400 (11.095 sec) INFO:tensorflow:loss = 44.54372, step = 400 (11.095 sec) INFO:tensorflow:global_step/sec: 8.952 INFO:tensorflow:global_step/sec: 8.952 INFO:tensorflow:loss = 35.568554, step = 500 (11.171 sec) INFO:tensorflow:loss = 35.568554, step = 500 (11.171 sec) INFO:tensorflow:global_step/sec: 9.09908 INFO:tensorflow:global_step/sec: 9.09908 INFO:tensorflow:loss = 42.5132, step = 600 (10.990 sec) INFO:tensorflow:loss = 42.5132, step = 600 (10.990 sec) INFO:tensorflow:global_step/sec: 9.02127 INFO:tensorflow:global_step/sec: 9.02127 INFO:tensorflow:loss = 40.52431, step = 700 (11.085 sec) INFO:tensorflow:loss = 40.52431, step = 700 (11.085 sec) INFO:tensorflow:global_step/sec: 9.09376 INFO:tensorflow:global_step/sec: 9.09376 INFO:tensorflow:loss = 37.5485, step = 800 (10.996 sec) INFO:tensorflow:loss = 37.5485, step = 800 (10.996 sec) INFO:tensorflow:global_step/sec: 9.11679 INFO:tensorflow:global_step/sec: 9.11679 INFO:tensorflow:loss = 32.65558, step = 900 (10.968 sec) INFO:tensorflow:loss = 32.65558, step = 900 (10.968 sec) INFO:tensorflow:Calling checkpoint listeners before saving checkpoint 1000... INFO:tensorflow:Calling checkpoint listeners before saving checkpoint 1000... INFO:tensorflow:Saving checkpoints for 1000 into /tmp/train/20220107-182759/model.ckpt. INFO:tensorflow:Saving checkpoints for 1000 into /tmp/train/20220107-182759/model.ckpt. INFO:tensorflow:Calling checkpoint listeners after saving checkpoint 1000... INFO:tensorflow:Calling checkpoint listeners after saving checkpoint 1000... INFO:tensorflow:Loss for final step: 46.92047. INFO:tensorflow:Loss for final step: 46.92047. INFO:tensorflow:Calling model_fn. INFO:tensorflow:Calling model_fn. 2022-01-07 18:30:32.176628: W tensorflow/core/common_runtime/graph_constructor.cc:1511] Importing a graph with a lower producer version 26 into an existing graph with producer version 987. Shape inference will have run different parts of the graph with different producer versions. INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Saver not created because there are no variables in the graph to restore INFO:tensorflow:Saver not created because there are no variables in the graph to restore WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead. WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead. WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead. WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to "careful_interpolation" instead. INFO:tensorflow:Done calling model_fn. INFO:tensorflow:Done calling model_fn. INFO:tensorflow:Signatures INCLUDED in export for Classify: None INFO:tensorflow:Signatures INCLUDED in export for Classify: None INFO:tensorflow:Signatures INCLUDED in export for Regress: None INFO:tensorflow:Signatures INCLUDED in export for Regress: None INFO:tensorflow:Signatures INCLUDED in export for Predict: None INFO:tensorflow:Signatures INCLUDED in export for Predict: None INFO:tensorflow:Signatures INCLUDED in export for Train: None INFO:tensorflow:Signatures INCLUDED in export for Train: None INFO:tensorflow:Signatures INCLUDED in export for Eval: ['eval'] INFO:tensorflow:Signatures INCLUDED in export for Eval: ['eval'] WARNING:tensorflow:Export includes no default signature! WARNING:tensorflow:Export includes no default signature! INFO:tensorflow:Restoring parameters from /tmp/train/20220107-182759/model.ckpt-1000 INFO:tensorflow:Restoring parameters from /tmp/train/20220107-182759/model.ckpt-1000 INFO:tensorflow:Assets added to graph. INFO:tensorflow:Assets added to graph. INFO:tensorflow:No assets to write. INFO:tensorflow:No assets to write. INFO:tensorflow:SavedModel written to: /tmp/tfma_eval_model/temp-1641580231/saved_model.pb INFO:tensorflow:SavedModel written to: /tmp/tfma_eval_model/temp-1641580231/saved_model.pb WARNING:absl:Tensorflow version (2.8.0-rc0) found. Note that TFMA support for TF 2.0 is currently in beta WARNING:root:Make sure that locally built Python SDK docker image has Python 3.7 interpreter. INFO:tensorflow:Restoring parameters from /tmp/tfma_eval_model/1641580231/variables/variables INFO:tensorflow:Restoring parameters from /tmp/tfma_eval_model/1641580231/variables/variables
widget_view.render_fairness_indicator(eval_result=eval_result_use)
FairnessIndicatorViewer(slicingMetrics=[{'sliceValue': 'Overall', 'slice': 'Overall', 'metrics': {'post_export…
การเปรียบเทียบการฝัง
คุณยังสามารถใช้ตัวบ่งชี้ความเป็นธรรมเพื่อเปรียบเทียบการฝังได้โดยตรง ตัวอย่างเช่น เปรียบเทียบแบบจำลองที่สร้างจากการฝัง NNLM และ USE
widget_view.render_fairness_indicator(multi_eval_results={'nnlm': eval_result_nnlm, 'use': eval_result_use})
FairnessIndicatorViewer(evalName='nnlm', evalNameCompare='use', slicingMetrics=[{'sliceValue': 'Overall', 'sli…