[root@nersc-tbn-6 ~]# graph_sweep.sh -t 56 -s /data/zettar/zx/src/sweep -b 16 -o /var/tmp -p -v
range_to_sweep  : 
threads         : 56
src_data_dir    : /data/zettar/zx/src/sweep
fs_block_size   : 4
block_size      : 16
num_sweep       : 3
output_dir      : /var/tmp
buffered        : 
verbose         : 1
push_button_plot: 1
dry_run         : 
===> Getting ready to sweep...
This is 0th sweep
range_to_sweep: 
threads       : 56
src_data_dir  : /data/zettar/zx/src/sweep
fs_block_size : 4
block_size    : 16m
buffered      : 
verbose       : 1
dry_run       : 
cmd           : 
Working on /data/zettar/zx/src/sweep/1048576x1KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       4945       5392
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :       4405      20107
          Total dirs       :          2         56
---
DROPCACHE Elapsed ms       :       3179       3738
---
WRITE     Elapsed ms       :      18608      19340
          Files/s          :      54247      54216
          Throughput MiB/s :         52         52
          Total files      :    1009448    1048600
          Total MiB        :        985       1024
---
DROPCACHE Elapsed ms       :      25034      25056
---
RMFILES   Elapsed ms       :      13183      14011
          Files/s          :      74773      74836
          Total files      :     985806    1048600
---
DROPCACHE Elapsed ms       :       5503       5536
---
Working on /data/zettar/zx/src/sweep/1048576x2KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       4746       5599
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      38961      42520
          Total dirs       :          6         56
---
DROPCACHE Elapsed ms       :       5055       5734
---
WRITE     Elapsed ms       :      18611      19193
          Files/s          :      54476      54634
          Throughput MiB/s :        106        106
          Total files      :    1013916    1048600
          Total MiB        :       1980       2048
---
DROPCACHE Elapsed ms       :      20773      20981
---
RMFILES   Elapsed ms       :      12775      13404
          Files/s          :      77443      78230
          Total files      :     989369    1048600
---
DROPCACHE Elapsed ms       :       5532       5553
---
Working on /data/zettar/zx/src/sweep/1048576x4KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       4543       5244
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      12269      31215
          Total dirs       :          2         56
---
DROPCACHE Elapsed ms       :       4529       5365
---
WRITE     Elapsed ms       :      19455      20020
          Files/s          :      52440      52375
          Throughput MiB/s :        204        204
          Total files      :    1020274    1048600
          Total MiB        :       3985       4096
---
DROPCACHE Elapsed ms       :      14849      14871
---
RMFILES   Elapsed ms       :      14067      14625
          Files/s          :      71220      71697
          Total files      :    1001921    1048600
---
DROPCACHE Elapsed ms       :       5518       5542
---
Working on /data/zettar/zx/src/sweep/1048576x8KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       4467       5007
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      21052      46627
          Total dirs       :          4         56
---
DROPCACHE Elapsed ms       :       3194       4152
---
WRITE     Elapsed ms       :      19279      20136
          Files/s          :      52210      52074
          Throughput MiB/s :        407        406
          Total files      :    1006598    1048600
          Total MiB        :       7864       8192
---
DROPCACHE Elapsed ms       :      15332      15342
---
RMFILES   Elapsed ms       :       6345      15913
          Files/s          :      49926      65892
          Total files      :     316806    1048600
---
DROPCACHE Elapsed ms       :       4935       4957
---
Working on /data/zettar/zx/src/sweep/1048576x16KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3423       3891
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      27397      32463
          Total dirs       :          4         56
---
DROPCACHE Elapsed ms       :       3379       3875
---
WRITE     Elapsed ms       :      18917      19702
          Files/s          :      53195      53222
          Throughput MiB/s :        831        831
          Total files      :    1006304    1048600
          Total MiB        :      15723      16384
---
DROPCACHE Elapsed ms       :      14269      14284
---
RMFILES   Elapsed ms       :       6840      15013
          Files/s          :      58868      69842
          Total files      :     402710    1048600
---
DROPCACHE Elapsed ms       :       5571       5595
---
Working on /data/zettar/zx/src/sweep/1048576x32KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3269       3806
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :       6644      28000
          Total dirs       :          2         56
---
DROPCACHE Elapsed ms       :       3498       4022
---
WRITE     Elapsed ms       :      18790      19610
          Files/s          :      53496      53470
          Throughput MiB/s :       1671       1670
          Total files      :    1005229    1048600
          Total MiB        :      31413      32768
---
DROPCACHE Elapsed ms       :      14192      14215
---
RMFILES   Elapsed ms       :       8838      13183
          Files/s          :      71813      79541
          Total files      :     634758    1048600
---
DROPCACHE Elapsed ms       :       5061       5085
---
Working on /data/zettar/zx/src/sweep/1048576x64KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3693       4293
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      51724      48611
          Total dirs       :          6         56
---
DROPCACHE Elapsed ms       :       3476       4406
---
WRITE     Elapsed ms       :      17971      19138
          Files/s          :      54844      54788
          Throughput MiB/s :       3427       3424
          Total files      :     985654    1048600
          Total MiB        :      61603      65537
---
DROPCACHE Elapsed ms       :      14764      14787
---
RMFILES   Elapsed ms       :      12110      12764
          Files/s          :      81015      82151
          Total files      :     981176    1048600
---
DROPCACHE Elapsed ms       :       5589       5620
---
Working on /data/zettar/zx/src/sweep/1048576x128KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       4846       5645
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      30120      30368
          Total dirs       :          5         56
---
DROPCACHE Elapsed ms       :       5009       5607
---
WRITE     Elapsed ms       :      18334      19060
          Files/s          :      55116      55015
          Throughput MiB/s :       6889       6876
          Total files      :    1010503    1048600
          Total MiB        :     126312     131075
---
DROPCACHE Elapsed ms       :      14236      14275
---
RMFILES   Elapsed ms       :      12297      15147
          Files/s          :      68576      69225
          Total files      :     843337    1048600
---
DROPCACHE Elapsed ms       :       5559       5581
---
Working on /data/zettar/zx/src/sweep/1048576x256KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       4869       5533
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      13513      42585
          Total dirs       :          2         56
---
DROPCACHE Elapsed ms       :       4321       4805
---
WRITE     Elapsed ms       :      25942      26611
          Files/s          :      39454      39404
          Throughput MiB/s :       9863       9851
          Total files      :    1023538    1048600
          Total MiB        :     255884     262150
---
DROPCACHE Elapsed ms       :      13812      13832
---
RMFILES   Elapsed ms       :       6442      13768
          Files/s          :      66271      76162
          Total files      :     426956    1048600
---
DROPCACHE Elapsed ms       :       5037       5057
---
Working on /data/zettar/zx/src/sweep/1048576x512KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       4073       4699
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      14388      40229
          Total dirs       :          2         56
---
DROPCACHE Elapsed ms       :       3858       4415
---
WRITE     Elapsed ms       :      57288      57970
          Files/s          :      18091      18088
          Throughput MiB/s :       9045       9044
          Total files      :    1036433    1048600
          Total MiB        :     518216     524300
---
DROPCACHE Elapsed ms       :      14077      14088
---
RMFILES   Elapsed ms       :       6338      13112
          Files/s          :      76616      79967
          Total files      :     485603    1048600
---
DROPCACHE Elapsed ms       :       5265       5287
---
Working on /data/zettar/zx/src/sweep/1048576x1MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3777       4258
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      54421      46052
          Total dirs       :          8         56
---
DROPCACHE Elapsed ms       :       3351       3854
---
WRITE     Elapsed ms       :     111488     111960
          Files/s          :       9368       9365
          Throughput MiB/s :       9368       9365
          Total files      :    1044490    1048600
          Total MiB        :    1044490    1048600
---
DROPCACHE Elapsed ms       :      14537      14552
---
RMFILES   Elapsed ms       :       5862      14753
          Files/s          :      62877      71075
          Total files      :     368587    1048600
---
DROPCACHE Elapsed ms       :       4813       4835
---
Working on /data/zettar/zx/src/sweep/524288x2MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       2756       4103
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      30303      33836
          Total dirs       :          5         56
---
DROPCACHE Elapsed ms       :       3506       4075
---
WRITE     Elapsed ms       :     111490     111765
          Files/s          :       4691       4691
          Throughput MiB/s :       9383       9382
          Total files      :     523083     524328
          Total MiB        :    1046166    1048656
---
DROPCACHE Elapsed ms       :       8931       8935
---
RMFILES   Elapsed ms       :       3153       6983
          Files/s          :      71341      75081
          Total files      :     224975     524328
---
DROPCACHE Elapsed ms       :       4151       4240
---
Working on /data/zettar/zx/src/sweep/262144x4MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3173       3690
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      41666      31354
          Total dirs       :          6         56
---
DROPCACHE Elapsed ms       :       2514       3783
---
WRITE     Elapsed ms       :     112014     112156
          Files/s          :       2337       2337
          Throughput MiB/s :       9350       9350
          Total files      :     261859     262192
          Total MiB        :    1047436    1048768
---
DROPCACHE Elapsed ms       :       6286       6288
---
RMFILES   Elapsed ms       :       1683       3585
          Files/s          :      69767      73127
          Total files      :     117478     262192
---
DROPCACHE Elapsed ms       :       3956       4364
---
Working on /data/zettar/zx/src/sweep/131072x8MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3725       4141
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      25974      24593
          Total dirs       :          4         56
---
DROPCACHE Elapsed ms       :       2977       3532
---
WRITE     Elapsed ms       :     107740     107897
          Files/s          :       1214       1215
          Throughput MiB/s :       9718       9720
          Total files      :     130888     131096
          Total MiB        :    1047104    1048768
---
DROPCACHE Elapsed ms       :       4529       4532
---
RMFILES   Elapsed ms       :        822       1765
          Files/s          :      71344      74256
          Total files      :      58670     131096
---
DROPCACHE Elapsed ms       :       3238       4218
---
Working on /data/zettar/zx/src/sweep/65536x16MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       2862       3986
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :       8097      29629
          Total dirs       :          2         56
---
DROPCACHE Elapsed ms       :       2454       4028
---
WRITE     Elapsed ms       :     102695     103041
          Files/s          :        636        636
          Throughput MiB/s :      10183      10182
          Total files      :      65361      65576
          Total MiB        :    1045776    1049216
---
DROPCACHE Elapsed ms       :       4139       4363
---
RMFILES   Elapsed ms       :        529        928
          Files/s          :      69008      70604
          Total files      :      36507      65576
---
DROPCACHE Elapsed ms       :       3552       4370
---
Working on /data/zettar/zx/src/sweep/32768x32MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3363       3958
---
MKDIRS    Elapsed ms       :          1          2
          Dirs/s           :       4587      18906
          Total dirs       :          7         56
---
DROPCACHE Elapsed ms       :       2837       4045
---
WRITE     Elapsed ms       :      99309      99430
          Files/s          :        329        330
          IOPS             :        659        660
          Throughput MiB/s :      10557      10561
          Total files      :      32736      32816
          Total MiB        :    1048416    1050112
---
DROPCACHE Elapsed ms       :       3791       4120
---
RMFILES   Elapsed ms       :        321        449
          Files/s          :      73468      73012
          Total files      :      23593      32816
---
DROPCACHE Elapsed ms       :       3627       4278
---
Working on /data/zettar/zx/src/sweep/16384x64MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3400       3986
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      29761      35828
          Total dirs       :          5         56
---
DROPCACHE Elapsed ms       :       3128       3996
---
WRITE     Elapsed ms       :     100408     100458
          Files/s          :        162        163
          IOPS             :        652        653
          Throughput MiB/s :      10447      10453
          Total files      :      16353      16408
          Total MiB        :    1049056    1050112
---
DROPCACHE Elapsed ms       :       3633       4020
---
RMFILES   Elapsed ms       :        212        240
          Files/s          :      68386      68214
          Total files      :      14512      16408
---
DROPCACHE Elapsed ms       :       3904       4395
---
Working on /data/zettar/zx/src/sweep/8192x128MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3587       4096
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      29761      31163
          Total dirs       :          5         56
---
DROPCACHE Elapsed ms       :       3494       4073
---
WRITE     Elapsed ms       :      96216      96413
          Files/s          :         84         85
          IOPS             :        682        683
          Throughput MiB/s :      10927      10928
          Total files      :       8177       8232
          Total MiB        :    1051424    1053696
---
DROPCACHE Elapsed ms       :       3728       4122
---
RMFILES   Elapsed ms       :        155        158
          Files/s          :      51807      51819
          Total files      :       8081       8232
---
DROPCACHE Elapsed ms       :       3655       4185
---
Working on /data/zettar/zx/src/sweep/4096x256MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3464       3902
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :       9740      30517
          Total dirs       :          3         56
---
DROPCACHE Elapsed ms       :       3394       3985
---
WRITE     Elapsed ms       :      89747      89832
          Files/s          :         45         46
          IOPS             :        737        738
          Throughput MiB/s :      11805      11809
          Total files      :       4089       4144
          Total MiB        :    1059472    1060864
---
DROPCACHE Elapsed ms       :       3783       4183
---
RMFILES   Elapsed ms       :        150        152
          Files/s          :      27047      27115
          Total files      :       4066       4144
---
DROPCACHE Elapsed ms       :       3945       4524
---
Working on /data/zettar/zx/src/sweep/2048x512MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3580       4020
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      26315      42977
          Total dirs       :          4         56
---
DROPCACHE Elapsed ms       :       3475       3975
---
WRITE     Elapsed ms       :     101240     101577
          Files/s          :         19         20
          IOPS             :        652        652
          Throughput MiB/s :      10442      10443
          Total files      :       2017       2072
          Total MiB        :    1057216    1060864
---
DROPCACHE Elapsed ms       :       3690       4109
---
RMFILES   Elapsed ms       :        142        146
          Files/s          :      13966      14183
          Total files      :       1988       2072
---
DROPCACHE Elapsed ms       :       3762       4178
---
Working on /data/zettar/zx/src/sweep/1024x1GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3622       4009
---
WRITE     Elapsed ms       :      89020     103269
          IOPS             :        634        634
          Throughput MiB/s :      10148      10153
          Total MiB        :     903472    1048576
---
DROPCACHE Elapsed ms       :       3324       3725
---
RMFILES   Elapsed ms       :         30         52
---
DROPCACHE Elapsed ms       :       3131       3623
---
Working on /data/zettar/zx/src/sweep/512x2GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3136       3611
---
WRITE     Elapsed ms       :      84594      99459
          IOPS             :        665        658
          Throughput MiB/s :      10645      10542
          Total MiB        :     900528    1048576
---
DROPCACHE Elapsed ms       :       3575       3980
---
RMFILES   Elapsed ms       :         16         27
---
DROPCACHE Elapsed ms       :       3240       3774
---
Working on /data/zettar/zx/src/sweep/256x4GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3040       3689
---
WRITE     Elapsed ms       :      92587     112943
          IOPS             :        585        580
          Throughput MiB/s :       9375       9284
          Total MiB        :     868032    1048576
---
DROPCACHE Elapsed ms       :       3779       4204
---
RMFILES   Elapsed ms       :          6         12
---
DROPCACHE Elapsed ms       :       3336       3943
---
Working on /data/zettar/zx/src/sweep/128x8GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       2981       3869
---
WRITE     Elapsed ms       :      76733     116556
          IOPS             :        575        562
          Throughput MiB/s :       9212       8996
          Total MiB        :     706944    1048576
---
DROPCACHE Elapsed ms       :       3815       4178
---
RMFILES   Elapsed ms       :          2          8
---
DROPCACHE Elapsed ms       :       2525       3692
---
Working on /data/zettar/zx/src/sweep/64x16GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3186       3579
---
WRITE     Elapsed ms       :      66257     105177
          IOPS             :        649        623
          Throughput MiB/s :      10392       9969
          Total MiB        :     688592    1048576
---
DROPCACHE Elapsed ms       :       3809       4152
---
RMFILES   Elapsed ms       :          1          4
---
DROPCACHE Elapsed ms       :       3156       3858
---
Working on /data/zettar/zx/src/sweep/32x32GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3442       3869
---
WRITE     Elapsed ms       :      88608     108259
          IOPS             :        602        605
          Throughput MiB/s :       9637       9685
          Total MiB        :     853952    1048576
---
DROPCACHE Elapsed ms       :       3601       3956
---
RMFILES   Elapsed ms       :          1          2
---
DROPCACHE Elapsed ms       :       3199       3703
---
Working on /data/zettar/zx/src/sweep/16x64GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3286       3644
---
WRITE     Elapsed ms       :      97675     108577
          IOPS             :        609        603
          Throughput MiB/s :       9744       9657
          Total MiB        :     951824    1048576
---
DROPCACHE Elapsed ms       :       3859       4261
---
RMFILES   Elapsed ms       :          0          1
---
DROPCACHE Elapsed ms       :       3473       4009
---
Working on /data/zettar/zx/src/sweep/8x128GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3403       3974
---
WRITE     Elapsed ms       :      99643     110529
          IOPS             :        594        592
          Throughput MiB/s :       9516       9486
          Total MiB        :     948224    1048576
---
DROPCACHE Elapsed ms       :       3622       3979
---
RMFILES   Elapsed ms       :          0          0
---
DROPCACHE Elapsed ms       :       3127       3742
---
Working on /data/zettar/zx/src/sweep/4x256GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3334       3825
---
WRITE     Elapsed ms       :      97094     108536
          IOPS             :        602        603
          Throughput MiB/s :       9639       9661
          Total MiB        :     935984    1048576
---
DROPCACHE Elapsed ms       :       3535       3888
---
RMFILES   Elapsed ms       :          0          0
---
DROPCACHE Elapsed ms       :       3228       3695
---
Working on /data/zettar/zx/src/sweep/2x512GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3132       3591
---
WRITE     Elapsed ms       :      46325      54691
          IOPS             :        594        599
          Throughput MiB/s :       9512       9586
          Total MiB        :     440688     524288
---
DROPCACHE Elapsed ms       :       3391       3722
---
RMFILES   Elapsed ms       :          0          1
---
DROPCACHE Elapsed ms       :       3152       3720
---
Working on /data/zettar/zx/src/sweep/1x1024GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3030       3571
---
WRITE     Elapsed ms       :      96719     110880
          IOPS             :        586        591
          Throughput MiB/s :       9376       9456
          Total MiB        :     906864    1048576
---
DROPCACHE Elapsed ms       :       3489       3804
---
RMFILES   Elapsed ms       :          0          1
---
DROPCACHE Elapsed ms       :       3299       3818
---
===> /usr/local/sbin/mtelbencho.sh all done :)
Total test time: 53m:39s
This is 1th sweep
range_to_sweep: 
threads       : 56
src_data_dir  : /data/zettar/zx/src/sweep
fs_block_size : 4
block_size    : 16m
buffered      : 
verbose       : 1
dry_run       : 
cmd           : 
Working on /data/zettar/zx/src/sweep/1048576x1KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3262       3891
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      14204      21068
          Total dirs       :          5         56
---
DROPCACHE Elapsed ms       :       3298       3952
---
WRITE     Elapsed ms       :      19038      19667
          Files/s          :      53208      53316
          Throughput MiB/s :         51         52
          Total files      :    1013039    1048600
          Total MiB        :        989       1024
---
DROPCACHE Elapsed ms       :      22892      23365
---
RMFILES   Elapsed ms       :      12617      13255
          Files/s          :      78460      79106
          Total files      :     990010    1048600
---
DROPCACHE Elapsed ms       :       5722       5766
---
Working on /data/zettar/zx/src/sweep/1048576x2KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       4792       5617
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      10256      22055
          Total dirs       :          4         56
---
DROPCACHE Elapsed ms       :       4993       5796
---
WRITE     Elapsed ms       :      18437      19385
          Files/s          :      53883      54090
          Throughput MiB/s :        105        105
          Total files      :     993451    1048600
          Total MiB        :       1940       2048
---
DROPCACHE Elapsed ms       :      20800      20832
---
RMFILES   Elapsed ms       :      13410      14333
          Files/s          :      72964      73155
          Total files      :     978488    1048600
---
DROPCACHE Elapsed ms       :       6052       6078
---
Working on /data/zettar/zx/src/sweep/1048576x4KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       4880       5622
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :       5347      26303
          Total dirs       :          2         56
---
DROPCACHE Elapsed ms       :       4891       5464
---
WRITE     Elapsed ms       :      19470      20517
          Files/s          :      51038      51106
          Throughput MiB/s :        199        199
          Total files      :     993759    1048600
          Total MiB        :       3881       4096
---
DROPCACHE Elapsed ms       :       7040      12544
---
RMFILES   Elapsed ms       :      10580      11602
          Files/s          :      90190      90376
          Total files      :     954232    1048600
---
DROPCACHE Elapsed ms       :       5796       5835
---
Working on /data/zettar/zx/src/sweep/1048576x8KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       4581       4967
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      47619      31284
          Total dirs       :         21         56
---
DROPCACHE Elapsed ms       :       3246       4038
---
WRITE     Elapsed ms       :      17965      19088
          Files/s          :      54856      54933
          Throughput MiB/s :        428        429
          Total files      :     985524    1048600
          Total MiB        :       7699       8192
---
DROPCACHE Elapsed ms       :      13923      13936
---
RMFILES   Elapsed ms       :       6732      14429
          Files/s          :      65776      72668
          Total files      :     442857    1048600
---
DROPCACHE Elapsed ms       :       4757       5034
---
Working on /data/zettar/zx/src/sweep/1048576x16KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3447       4585
---
MKDIRS    Elapsed ms       :          0          3
          Dirs/s           :       5524      15681
          Total dirs       :          2         56
---
DROPCACHE Elapsed ms       :       3765       4266
---
WRITE     Elapsed ms       :      20691      21497
          Files/s          :      48665      48778
          Throughput MiB/s :        760        762
          Total files      :    1006966    1048600
          Total MiB        :      15733      16384
---
DROPCACHE Elapsed ms       :      13971      13987
---
RMFILES   Elapsed ms       :       6915      13294
          Files/s          :      69576      78873
          Total files      :     481173    1048600
---
DROPCACHE Elapsed ms       :       5223       5244
---
Working on /data/zettar/zx/src/sweep/1048576x32KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       4082       4548
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      24475      30123
          Total dirs       :          7         56
---
DROPCACHE Elapsed ms       :       3779       4264
---
WRITE     Elapsed ms       :      20136      21281
          Files/s          :      49518      49273
          Throughput MiB/s :       1547       1539
          Total files      :     997105    1048600
          Total MiB        :      31159      32768
---
DROPCACHE Elapsed ms       :      13575      13579
---
RMFILES   Elapsed ms       :       6502      13294
          Files/s          :      73839      78874
          Total files      :     480158    1048600
---
DROPCACHE Elapsed ms       :       5083       5107
---
Working on /data/zettar/zx/src/sweep/1048576x64KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3926       4373
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :       6493      28527
          Total dirs       :          2         56
---
DROPCACHE Elapsed ms       :       3468       3967
---
WRITE     Elapsed ms       :      19102      20187
          Files/s          :      51928      51944
          Throughput MiB/s :       3245       3246
          Total files      :     991936    1048600
          Total MiB        :      61996      65537
---
DROPCACHE Elapsed ms       :      13938      13952
---
RMFILES   Elapsed ms       :       4783      14074
          Files/s          :      52886      74502
          Total files      :     252966    1048600
---
DROPCACHE Elapsed ms       :       4887       4914
---
Working on /data/zettar/zx/src/sweep/1048576x128KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3455       3895
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :       3558      33755
          Total dirs       :          1         56
---
DROPCACHE Elapsed ms       :       3041       3503
---
WRITE     Elapsed ms       :      18747      19312
          Files/s          :      54289      54295
          Throughput MiB/s :       6786       6786
          Total files      :    1017767    1048600
          Total MiB        :     127220     131075
---
DROPCACHE Elapsed ms       :      13168      13213
---
RMFILES   Elapsed ms       :       4076      13979
          Files/s          :      56223      75008
          Total files      :     229194    1048600
---
DROPCACHE Elapsed ms       :       5002       5028
---
Working on /data/zettar/zx/src/sweep/1048576x256KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       1054       4013
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :       6036      25699
          Total dirs       :          3         56
---
DROPCACHE Elapsed ms       :       2304       4262
---
WRITE     Elapsed ms       :      28545      29041
          Files/s          :      36144      36106
          Throughput MiB/s :       9036       9026
          Total files      :    1031768    1048600
          Total MiB        :     257942     262150
---
DROPCACHE Elapsed ms       :      14461      14468
---
RMFILES   Elapsed ms       :       5045      15165
          Files/s          :      53385      69143
          Total files      :     269347    1048600
---
DROPCACHE Elapsed ms       :       4987       5013
---
Working on /data/zettar/zx/src/sweep/1048576x512KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       2740       4258
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      14527      22391
          Total dirs       :          6         56
---
DROPCACHE Elapsed ms       :       2067       4145
---
WRITE     Elapsed ms       :      57219      57739
          Files/s          :      18152      18160
          Throughput MiB/s :       9076       9080
          Total files      :    1038655    1048600
          Total MiB        :     519327     524300
---
DROPCACHE Elapsed ms       :      12991      13003
---
RMFILES   Elapsed ms       :       7110      13951
          Files/s          :      70635      75161
          Total files      :     502243    1048600
---
DROPCACHE Elapsed ms       :       2386       4518
---
Working on /data/zettar/zx/src/sweep/1048576x1MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3349       4220
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      33444      32710
          Total dirs       :         10         56
---
DROPCACHE Elapsed ms       :       1168       4010
---
WRITE     Elapsed ms       :     113237     113678
          Files/s          :       9225       9224
          Throughput MiB/s :       9225       9224
          Total files      :    1044638    1048600
          Total MiB        :    1044638    1048600
---
DROPCACHE Elapsed ms       :      14215      14227
---
RMFILES   Elapsed ms       :       5576      13844
          Files/s          :      69970      75743
          Total files      :     390180    1048600
---
DROPCACHE Elapsed ms       :       4556       4577
---
Working on /data/zettar/zx/src/sweep/524288x2MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       1160       3807
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      20779      21806
          Total dirs       :          8         56
---
DROPCACHE Elapsed ms       :       3630       4178
---
WRITE     Elapsed ms       :     113103     113337
          Files/s          :       4624       4626
          Throughput MiB/s :       9249       9252
          Total files      :     523102     524328
          Total MiB        :    1046204    1048656
---
DROPCACHE Elapsed ms       :       7904       7937
---
RMFILES   Elapsed ms       :       2903       6857
          Files/s          :      78489      76465
          Total files      :     227878     524328
---
DROPCACHE Elapsed ms       :       3811       4577
---
Working on /data/zettar/zx/src/sweep/262144x4MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3012       4317
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :       7874      36721
          Total dirs       :          3         56
---
DROPCACHE Elapsed ms       :       3510       4016
---
WRITE     Elapsed ms       :     116085     116282
          Files/s          :       2254       2254
          Throughput MiB/s :       9017       9019
          Total files      :     261699     262192
          Total MiB        :    1046796    1048768
---
DROPCACHE Elapsed ms       :       5485       5527
---
RMFILES   Elapsed ms       :       1545       3259
          Files/s          :      77710      80435
          Total files      :     120074     262192
---
DROPCACHE Elapsed ms       :       4185       4581
---
Working on /data/zettar/zx/src/sweep/131072x8MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3798       4356
---
MKDIRS    Elapsed ms       :          0          3
          Dirs/s           :       2583      15868
          Total dirs       :          1         56
---
DROPCACHE Elapsed ms       :       3834       4382
---
WRITE     Elapsed ms       :     105268     105401
          Files/s          :       1243       1243
          Throughput MiB/s :       9948       9950
          Total files      :     130912     131096
          Total MiB        :    1047296    1048768
---
DROPCACHE Elapsed ms       :       4127       4157
---
RMFILES   Elapsed ms       :        728       1664
          Files/s          :      74357      78757
          Total files      :      54178     131096
---
DROPCACHE Elapsed ms       :       3582       4063
---
Working on /data/zettar/zx/src/sweep/65536x16MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3588       4047
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :       2118      18893
          Total dirs       :          1         56
---
DROPCACHE Elapsed ms       :       3620       4188
---
WRITE     Elapsed ms       :     106490     106808
          Files/s          :        613        613
          Throughput MiB/s :       9816       9823
          Total files      :      65333      65576
          Total MiB        :    1045328    1049216
---
DROPCACHE Elapsed ms       :       3783       3977
---
RMFILES   Elapsed ms       :        404        818
          Files/s          :      78191      80137
          Total files      :      31631      65576
---
DROPCACHE Elapsed ms       :       3199       4013
---
Working on /data/zettar/zx/src/sweep/32768x32MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3241       3688
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :       6896      34674
          Total dirs       :          2         56
---
DROPCACHE Elapsed ms       :       2791       3839
---
WRITE     Elapsed ms       :      95405      95755
          Files/s          :        342        342
          IOPS             :        685        685
          Throughput MiB/s :      10968      10966
          Total files      :      32692      32816
          Total MiB        :    1046496    1050112
---
DROPCACHE Elapsed ms       :       3771       4107
---
RMFILES   Elapsed ms       :        302        419
          Files/s          :      77490      78167
          Total files      :      23419      32816
---
DROPCACHE Elapsed ms       :       3277       3892
---
Working on /data/zettar/zx/src/sweep/16384x64MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3264       3704
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      33333      26540
          Total dirs       :         10         56
---
DROPCACHE Elapsed ms       :       3217       3745
---
WRITE     Elapsed ms       :     102146     102390
          Files/s          :        159        160
          IOPS             :        640        640
          Throughput MiB/s :      10251      10255
          Total files      :      16340      16408
          Total MiB        :    1047184    1050112
---
DROPCACHE Elapsed ms       :       4091       4426
---
RMFILES   Elapsed ms       :        232        256
          Files/s          :      64043      63861
          Total files      :      14859      16408
---
DROPCACHE Elapsed ms       :       4157       4675
---
Working on /data/zettar/zx/src/sweep/8192x128MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3678       4202
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :       9803      21148
          Total dirs       :          4         56
---
DROPCACHE Elapsed ms       :       3365       3822
---
WRITE     Elapsed ms       :      95220      95460
          Files/s          :         85         86
          IOPS             :        689        689
          Throughput MiB/s :      11033      11038
          Total files      :       8177       8232
          Total MiB        :    1050576    1053696
---
DROPCACHE Elapsed ms       :       3526       3913
---
RMFILES   Elapsed ms       :        150        153
          Files/s          :      53513      53482
          Total files      :       8064       8232
---
DROPCACHE Elapsed ms       :       3530       3915
---
Working on /data/zettar/zx/src/sweep/4096x256MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3067       3695
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      30303      24866
          Total dirs       :          9         56
---
DROPCACHE Elapsed ms       :       3142       3656
---
WRITE     Elapsed ms       :     100330     100522
          Files/s          :         40         41
          IOPS             :        659        659
          Throughput MiB/s :      10545      10553
          Total files      :       4089       4144
          Total MiB        :    1058064    1060864
---
DROPCACHE Elapsed ms       :       3467       3848
---
RMFILES   Elapsed ms       :        145        148
          Files/s          :      27918      27914
          Total files      :       4067       4144
---
DROPCACHE Elapsed ms       :       3548       4014
---
Working on /data/zettar/zx/src/sweep/2048x512MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3459       3931
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      44487      25747
          Total dirs       :         23         56
---
DROPCACHE Elapsed ms       :       3527       4085
---
WRITE     Elapsed ms       :     105223     105289
          Files/s          :         19         19
          IOPS             :        629        629
          Throughput MiB/s :      10068      10075
          Total files      :       2018       2072
          Total MiB        :    1059456    1060864
---
DROPCACHE Elapsed ms       :       3445       3821
---
RMFILES   Elapsed ms       :        136        140
          Files/s          :      14607      14759
          Total files      :       2001       2072
---
DROPCACHE Elapsed ms       :       3680       4170
---
Working on /data/zettar/zx/src/sweep/1024x1GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3507       3956
---
WRITE     Elapsed ms       :      85705      99356
          IOPS             :        661        659
          Throughput MiB/s :      10577      10553
          Total MiB        :     906592    1048576
---
DROPCACHE Elapsed ms       :       3506       3885
---
RMFILES   Elapsed ms       :         36         50
---
DROPCACHE Elapsed ms       :       3174       3605
---
Working on /data/zettar/zx/src/sweep/512x2GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3126       3514
---
WRITE     Elapsed ms       :      80527      96739
          IOPS             :        695        677
          Throughput MiB/s :      11124      10839
          Total MiB        :     895840    1048576
---
DROPCACHE Elapsed ms       :       3859       4260
---
RMFILES   Elapsed ms       :         16         29
---
DROPCACHE Elapsed ms       :       3317       3858
---
Working on /data/zettar/zx/src/sweep/256x4GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3033       3552
---
WRITE     Elapsed ms       :      83812     103318
          IOPS             :        642        634
          Throughput MiB/s :      10281      10148
          Total MiB        :     861712    1048576
---
DROPCACHE Elapsed ms       :       3533       3899
---
RMFILES   Elapsed ms       :          6         13
---
DROPCACHE Elapsed ms       :       2980       3708
---
Working on /data/zettar/zx/src/sweep/128x8GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3252       3650
---
WRITE     Elapsed ms       :      72166     113419
          IOPS             :        595        577
          Throughput MiB/s :       9532       9245
          Total MiB        :     687936    1048576
---
DROPCACHE Elapsed ms       :       3874       4258
---
RMFILES   Elapsed ms       :          3          8
---
DROPCACHE Elapsed ms       :       2676       3998
---
Working on /data/zettar/zx/src/sweep/64x16GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3319       3892
---
WRITE     Elapsed ms       :      82736     117875
          IOPS             :        560        555
          Throughput MiB/s :       8970       8895
          Total MiB        :     742176    1048576
---
DROPCACHE Elapsed ms       :       3956       4254
---
RMFILES   Elapsed ms       :          2          4
---
DROPCACHE Elapsed ms       :       3472       4011
---
Working on /data/zettar/zx/src/sweep/32x32GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3365       3839
---
WRITE     Elapsed ms       :      92394     111227
          IOPS             :        584        589
          Throughput MiB/s :       9349       9427
          Total MiB        :     863872    1048576
---
DROPCACHE Elapsed ms       :       3558       3875
---
RMFILES   Elapsed ms       :          1          2
---
DROPCACHE Elapsed ms       :       3090       3751
---
Working on /data/zettar/zx/src/sweep/16x64GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3338       3933
---
WRITE     Elapsed ms       :     102063     110952
          IOPS             :        587        590
          Throughput MiB/s :       9399       9450
          Total MiB        :     959344    1048576
---
DROPCACHE Elapsed ms       :       3487       3864
---
RMFILES   Elapsed ms       :          0          1
---
DROPCACHE Elapsed ms       :       3126       3590
---
Working on /data/zettar/zx/src/sweep/8x128GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3009       3589
---
WRITE     Elapsed ms       :     100158     107675
          IOPS             :        608        608
          Throughput MiB/s :       9728       9738
          Total MiB        :     974384    1048576
---
DROPCACHE Elapsed ms       :       3519       3841
---
RMFILES   Elapsed ms       :          0          0
---
DROPCACHE Elapsed ms       :       3010       3505
---
Working on /data/zettar/zx/src/sweep/4x256GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       2717       3439
---
WRITE     Elapsed ms       :      96875     109763
          IOPS             :        609        597
          Throughput MiB/s :       9749       9553
          Total MiB        :     944448    1048576
---
DROPCACHE Elapsed ms       :       3999       4387
---
RMFILES   Elapsed ms       :          0          1
---
DROPCACHE Elapsed ms       :       3197       3937
---
Working on /data/zettar/zx/src/sweep/2x512GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3535       3957
---
WRITE     Elapsed ms       :      45890      56731
          IOPS             :        573        577
          Throughput MiB/s :       9181       9241
          Total MiB        :     421344     524288
---
DROPCACHE Elapsed ms       :       3258       3638
---
RMFILES   Elapsed ms       :          0          1
---
DROPCACHE Elapsed ms       :       2960       3448
---
Working on /data/zettar/zx/src/sweep/1x1024GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3065       3535
---
WRITE     Elapsed ms       :      95677     111570
          IOPS             :        592        587
          Throughput MiB/s :       9479       9398
          Total MiB        :     907008    1048576
---
DROPCACHE Elapsed ms       :       3960       4288
---
RMFILES   Elapsed ms       :          0          0
---
DROPCACHE Elapsed ms       :       3101       3693
---
===> /usr/local/sbin/mtelbencho.sh all done :)
Total test time: 53m:40s
This is 2th sweep
range_to_sweep: 
threads       : 56
src_data_dir  : /data/zettar/zx/src/sweep
fs_block_size : 4
block_size    : 16m
buffered      : 
verbose       : 1
dry_run       : 
cmd           : 
Working on /data/zettar/zx/src/sweep/1048576x1KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3179       3869
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :       9708      19015
          Total dirs       :          4         56
---
DROPCACHE Elapsed ms       :       3288       3966
---
WRITE     Elapsed ms       :      19250      19965
          Files/s          :      52469      52519
          Throughput MiB/s :         51         51
          Total files      :    1010072    1048600
          Total MiB        :        986       1024
---
DROPCACHE Elapsed ms       :      21394      22182
---
RMFILES   Elapsed ms       :      12402      12896
          Files/s          :      80263      81309
          Total files      :     995457    1048600
---
DROPCACHE Elapsed ms       :       4842       4866
---
Working on /data/zettar/zx/src/sweep/1048576x2KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3316       3800
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :       9584      34146
          Total dirs       :          3         56
---
DROPCACHE Elapsed ms       :       3481       3956
---
WRITE     Elapsed ms       :      18702      19318
          Files/s          :      54206      54279
          Throughput MiB/s :        105        106
          Total files      :    1013786    1048600
          Total MiB        :       1980       2048
---
DROPCACHE Elapsed ms       :      16927      16948
---
RMFILES   Elapsed ms       :      12837      13302
          Files/s          :      77966      78826
          Total files      :    1000870    1048600
---
DROPCACHE Elapsed ms       :       4422       4491
---
Working on /data/zettar/zx/src/sweep/1048576x4KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       1241       4185
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      30120      23779
          Total dirs       :         10         56
---
DROPCACHE Elapsed ms       :       3181       4031
---
WRITE     Elapsed ms       :      18324      19144
          Files/s          :      54915      54772
          Throughput MiB/s :        214        213
          Total files      :    1006314    1048600
          Total MiB        :       3930       4096
---
DROPCACHE Elapsed ms       :      14325      14373
---
RMFILES   Elapsed ms       :       6507      14820
          Files/s          :      58287      70753
          Total files      :     379290    1048600
---
DROPCACHE Elapsed ms       :       5120       5144
---
Working on /data/zettar/zx/src/sweep/1048576x8KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3437       3992
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      33259      33116
          Total dirs       :         15         56
---
DROPCACHE Elapsed ms       :       3470       4012
---
WRITE     Elapsed ms       :      18269      19113
          Files/s          :      55018      54861
          Throughput MiB/s :        429        428
          Total files      :    1005167    1048600
          Total MiB        :       7852       8192
---
DROPCACHE Elapsed ms       :      14292      14312
---
RMFILES   Elapsed ms       :      11363      13406
          Files/s          :      76072      78213
          Total files      :     864471    1048600
---
DROPCACHE Elapsed ms       :       5753       5772
---
Working on /data/zettar/zx/src/sweep/1048576x16KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       4163       4648
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      52631      24179
          Total dirs       :         15         56
---
DROPCACHE Elapsed ms       :       3572       4096
---
WRITE     Elapsed ms       :      18022      19075
          Files/s          :      55196      54971
          Throughput MiB/s :        862        858
          Total files      :     994780    1048600
          Total MiB        :      15543      16384
---
DROPCACHE Elapsed ms       :      15942      15955
---
RMFILES   Elapsed ms       :       8177      17015
          Files/s          :      52272      61624
          Total files      :     427443    1048600
---
DROPCACHE Elapsed ms       :       5399       5426
---
Working on /data/zettar/zx/src/sweep/1048576x32KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3691       4248
---
MKDIRS    Elapsed ms       :          2          2
          Dirs/s           :      21604      20717
          Total dirs       :         56         56
---
DROPCACHE Elapsed ms       :       3575       4245
---
WRITE     Elapsed ms       :      19563      20400
          Files/s          :      51413      51399
          Throughput MiB/s :       1606       1606
          Total files      :    1005820    1048600
          Total MiB        :      31431      32768
---
DROPCACHE Elapsed ms       :       6686      13230
---
RMFILES   Elapsed ms       :      11219      12977
          Files/s          :      79016      80803
          Total files      :     886552    1048600
---
DROPCACHE Elapsed ms       :       5510       5532
---
Working on /data/zettar/zx/src/sweep/1048576x64KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       4572       5142
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      18237      29136
          Total dirs       :          6         56
---
DROPCACHE Elapsed ms       :       4311       4669
---
WRITE     Elapsed ms       :      19068      20015
          Files/s          :      52371      52388
          Throughput MiB/s :       3273       3274
          Total files      :     998663    1048600
          Total MiB        :      62416      65537
---
DROPCACHE Elapsed ms       :      15126      15132
---
RMFILES   Elapsed ms       :       7751      15987
          Files/s          :      55809      65588
          Total files      :     432578    1048600
---
DROPCACHE Elapsed ms       :       5167       5189
---
Working on /data/zettar/zx/src/sweep/1048576x128KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3545       4026
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      17391      20289
          Total dirs       :          6         56
---
DROPCACHE Elapsed ms       :       3609       4147
---
WRITE     Elapsed ms       :      19609      20202
          Files/s          :      51972      51905
          Throughput MiB/s :       6496       6488
          Total files      :    1019165    1048600
          Total MiB        :     127395     131075
---
DROPCACHE Elapsed ms       :      14416      14441
---
RMFILES   Elapsed ms       :       4775      14638
          Files/s          :      53738      71631
          Total files      :     256654    1048600
---
DROPCACHE Elapsed ms       :       5036       5058
---
Working on /data/zettar/zx/src/sweep/1048576x256KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3305       3719
---
MKDIRS    Elapsed ms       :          1          1
          Dirs/s           :      35087      34504
          Total dirs       :         56         56
---
DROPCACHE Elapsed ms       :       2857       3442
---
WRITE     Elapsed ms       :      26670      27266
          Files/s          :      38495      38457
          Throughput MiB/s :       9623       9614
          Total files      :    1026693    1048600
          Total MiB        :     256673     262150
---
DROPCACHE Elapsed ms       :      13290      13322
---
RMFILES   Elapsed ms       :       7852      12797
          Files/s          :      77183      81937
          Total files      :     606112    1048600
---
DROPCACHE Elapsed ms       :       4842       4911
---
Working on /data/zettar/zx/src/sweep/1048576x512KiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3948       4475
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      26119      24158
          Total dirs       :          7         56
---
DROPCACHE Elapsed ms       :       3618       4185
---
WRITE     Elapsed ms       :      55531      56088
          Files/s          :      18696      18695
          Throughput MiB/s :       9348       9347
          Total files      :    1038249    1048600
          Total MiB        :     519124     524300
---
DROPCACHE Elapsed ms       :      13096      13108
---
RMFILES   Elapsed ms       :       4037      13223
          Files/s          :      73515      79299
          Total files      :     296836    1048600
---
DROPCACHE Elapsed ms       :       5206       5228
---
Working on /data/zettar/zx/src/sweep/1048576x1MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3852       4289
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      10273      31692
          Total dirs       :          3         56
---
DROPCACHE Elapsed ms       :       3633       4119
---
WRITE     Elapsed ms       :     108241     108518
          Files/s          :       9662       9662
          Throughput MiB/s :       9662       9662
          Total files      :    1045909    1048600
          Total MiB        :    1045909    1048600
---
DROPCACHE Elapsed ms       :      13780      13812
---
RMFILES   Elapsed ms       :       4010      13678
          Files/s          :      68129      76661
          Total files      :     273238    1048600
---
DROPCACHE Elapsed ms       :       5050       5080
---
Working on /data/zettar/zx/src/sweep/524288x2MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3974       4355
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      26119      21926
          Total dirs       :          7         56
---
DROPCACHE Elapsed ms       :       3419       3945
---
WRITE     Elapsed ms       :     116200     116570
          Files/s          :       4499       4497
          Throughput MiB/s :       8998       8995
          Total files      :     522810     524328
          Total MiB        :    1045620    1048656
---
DROPCACHE Elapsed ms       :       9519       9522
---
RMFILES   Elapsed ms       :       4096       7627
          Files/s          :      67090      68737
          Total files      :     274843     524328
---
DROPCACHE Elapsed ms       :       4763       4887
---
Working on /data/zettar/zx/src/sweep/262144x4MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       4213       4676
---
MKDIRS    Elapsed ms       :          0          3
          Dirs/s           :       5730      17643
          Total dirs       :          2         56
---
DROPCACHE Elapsed ms       :       3930       4462
---
WRITE     Elapsed ms       :     109343     109567
          Files/s          :       2392       2392
          Throughput MiB/s :       9571       9571
          Total files      :     261644     262192
          Total MiB        :    1046576    1048768
---
DROPCACHE Elapsed ms       :       6156       6157
---
RMFILES   Elapsed ms       :       1869       3446
          Files/s          :      72833      76064
          Total files      :     136144     262192
---
DROPCACHE Elapsed ms       :       2294       3763
---
Working on /data/zettar/zx/src/sweep/131072x8MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       2844       3664
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :       6042      21638
          Total dirs       :          2         56
---
DROPCACHE Elapsed ms       :       1007       3761
---
WRITE     Elapsed ms       :     104019     104138
          Files/s          :       1258       1258
          Throughput MiB/s :      10067      10070
          Total files      :     130899     131096
          Total MiB        :    1047192    1048768
---
DROPCACHE Elapsed ms       :       4239       4273
---
RMFILES   Elapsed ms       :        858       1666
          Files/s          :      74408      78668
          Total files      :      63908     131096
---
DROPCACHE Elapsed ms       :       3560       4122
---
Working on /data/zettar/zx/src/sweep/65536x16MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3498       3975
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      10600      20756
          Total dirs       :          3         56
---
DROPCACHE Elapsed ms       :       3448       3881
---
WRITE     Elapsed ms       :     100440     100634
          Files/s          :        651        651
          Throughput MiB/s :      10423      10425
          Total files      :      65436      65576
          Total MiB        :    1046976    1049216
---
DROPCACHE Elapsed ms       :       3818       4048
---
RMFILES   Elapsed ms       :        403        864
          Files/s          :      74715      75827
          Total files      :      30136      65576
---
DROPCACHE Elapsed ms       :       3763       4320
---
Working on /data/zettar/zx/src/sweep/32768x32MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3699       4068
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      13605      19649
          Total dirs       :          4         56
---
DROPCACHE Elapsed ms       :       3420       3939
---
WRITE     Elapsed ms       :     105361     105615
          Files/s          :        310        310
          IOPS             :        621        621
          Throughput MiB/s :       9942       9942
          Total files      :      32725      32816
          Total MiB        :    1047584    1050112
---
DROPCACHE Elapsed ms       :       3921       4309
---
RMFILES   Elapsed ms       :        323        485
          Files/s          :      66297      67575
          Total files      :      21465      32816
---
DROPCACHE Elapsed ms       :       3989       4554
---
Working on /data/zettar/zx/src/sweep/16384x64MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3557       4158
---
MKDIRS    Elapsed ms       :          0          3
          Dirs/s           :       3067      17660
          Total dirs       :          2         56
---
DROPCACHE Elapsed ms       :       3695       4267
---
WRITE     Elapsed ms       :     101641     101912
          Files/s          :        160        161
          IOPS             :        643        644
          Throughput MiB/s :      10300      10304
          Total files      :      16353      16408
          Total MiB        :    1046928    1050112
---
DROPCACHE Elapsed ms       :       3753       4091
---
RMFILES   Elapsed ms       :        226        247
          Files/s          :      66021      66428
          Total files      :      14921      16408
---
DROPCACHE Elapsed ms       :       3967       4494
---
Working on /data/zettar/zx/src/sweep/8192x128MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3522       3989
---
MKDIRS    Elapsed ms       :          0          1
          Dirs/s           :      19774      30922
          Total dirs       :          7         56
---
DROPCACHE Elapsed ms       :       3339       3869
---
WRITE     Elapsed ms       :     102957     103087
          Files/s          :         79         79
          IOPS             :        638        638
          Throughput MiB/s :      10215      10221
          Total files      :       8177       8232
          Total MiB        :    1051760    1053696
---
DROPCACHE Elapsed ms       :       3832       4216
---
RMFILES   Elapsed ms       :        162        166
          Files/s          :      49264      49503
          Total files      :       7993       8232
---
DROPCACHE Elapsed ms       :       3716       4267
---
Working on /data/zettar/zx/src/sweep/4096x256MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3404       3780
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      23333      23972
          Total dirs       :          7         56
---
DROPCACHE Elapsed ms       :       3224       3724
---
WRITE     Elapsed ms       :      92629      92838
          Files/s          :         44         44
          IOPS             :        713        714
          Throughput MiB/s :      11422      11426
          Total files      :       4089       4144
          Total MiB        :    1058032    1060864
---
DROPCACHE Elapsed ms       :       3808       4134
---
RMFILES   Elapsed ms       :        141        144
          Files/s          :      28374      28580
          Total files      :       4021       4144
---
DROPCACHE Elapsed ms       :       3596       4087
---
Working on /data/zettar/zx/src/sweep/2048x512MiB with 56 threads...
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3455       3863
---
MKDIRS    Elapsed ms       :          0          2
          Dirs/s           :      16286      25089
          Total dirs       :          5         56
---
DROPCACHE Elapsed ms       :       3360       3877
---
WRITE     Elapsed ms       :     101180     101401
          Files/s          :         19         20
          IOPS             :        653        653
          Throughput MiB/s :      10456      10462
          Total files      :       2017       2072
          Total MiB        :    1058032    1060864
---
DROPCACHE Elapsed ms       :       3755       4131
---
RMFILES   Elapsed ms       :        168        171
          Files/s          :      11938      12100
          Total files      :       2013       2072
---
DROPCACHE Elapsed ms       :       3680       4268
---
Working on /data/zettar/zx/src/sweep/1024x1GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3533       3911
---
WRITE     Elapsed ms       :      83778      98708
          IOPS             :        675        663
          Throughput MiB/s :      10803      10623
          Total MiB        :     905104    1048576
---
DROPCACHE Elapsed ms       :       3697       4058
---
RMFILES   Elapsed ms       :         35         52
---
DROPCACHE Elapsed ms       :       3330       3857
---
Working on /data/zettar/zx/src/sweep/512x2GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3541       3956
---
WRITE     Elapsed ms       :      76928      91520
          IOPS             :        729        716
          Throughput MiB/s :      11669      11457
          Total MiB        :     897744    1048576
---
DROPCACHE Elapsed ms       :       3819       4243
---
RMFILES   Elapsed ms       :         15         27
---
DROPCACHE Elapsed ms       :       3455       3978
---
Working on /data/zettar/zx/src/sweep/256x4GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3492       3926
---
WRITE     Elapsed ms       :      87873     109117
          IOPS             :        607        600
          Throughput MiB/s :       9727       9609
          Total MiB        :     854800    1048576
---
DROPCACHE Elapsed ms       :       3696       4094
---
RMFILES   Elapsed ms       :          7         13
---
DROPCACHE Elapsed ms       :       3487       3986
---
Working on /data/zettar/zx/src/sweep/128x8GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3599       4068
---
WRITE     Elapsed ms       :      72653     111440
          IOPS             :        599        588
          Throughput MiB/s :       9585       9409
          Total MiB        :     696384    1048576
---
DROPCACHE Elapsed ms       :       3841       4239
---
RMFILES   Elapsed ms       :          1          7
---
DROPCACHE Elapsed ms       :       2792       3907
---
Working on /data/zettar/zx/src/sweep/64x16GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3883       4310
---
WRITE     Elapsed ms       :      70322     109653
          IOPS             :        614        597
          Throughput MiB/s :       9836       9562
          Total MiB        :     691728    1048576
---
DROPCACHE Elapsed ms       :       3802       4222
---
RMFILES   Elapsed ms       :          3          4
---
DROPCACHE Elapsed ms       :       3427       3947
---
Working on /data/zettar/zx/src/sweep/32x32GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3295       3781
---
WRITE     Elapsed ms       :      83565     109302
          IOPS             :        609        599
          Throughput MiB/s :       9749       9593
          Total MiB        :     814688    1048576
---
DROPCACHE Elapsed ms       :       3923       4278
---
RMFILES   Elapsed ms       :          1          2
---
DROPCACHE Elapsed ms       :       3345       3895
---
Working on /data/zettar/zx/src/sweep/16x64GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3564       3972
---
WRITE     Elapsed ms       :      97523     108187
          IOPS             :        607        605
          Throughput MiB/s :       9726       9692
          Total MiB        :     948528    1048576
---
DROPCACHE Elapsed ms       :       3752       4147
---
RMFILES   Elapsed ms       :          0          1
---
DROPCACHE Elapsed ms       :       3331       3888
---
Working on /data/zettar/zx/src/sweep/8x128GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3378       3842
---
WRITE     Elapsed ms       :      95756     107435
          IOPS             :        614        610
          Throughput MiB/s :       9824       9760
          Total MiB        :     940720    1048576
---
DROPCACHE Elapsed ms       :       3641       4056
---
RMFILES   Elapsed ms       :          0          0
---
DROPCACHE Elapsed ms       :       3269       3955
---
Working on /data/zettar/zx/src/sweep/4x256GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3277       3761
---
WRITE     Elapsed ms       :     102036     113874
          IOPS             :        575        575
          Throughput MiB/s :       9211       9208
          Total MiB        :     939936    1048576
---
DROPCACHE Elapsed ms       :       3611       4020
---
RMFILES   Elapsed ms       :          0          1
---
DROPCACHE Elapsed ms       :       3354       3826
---
Working on /data/zettar/zx/src/sweep/2x512GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3355       3773
---
WRITE     Elapsed ms       :      45998      55131
          IOPS             :        596        594
          Throughput MiB/s :       9541       9509
          Total MiB        :     438896     524288
---
DROPCACHE Elapsed ms       :       3740       4150
---
RMFILES   Elapsed ms       :          0          1
---
DROPCACHE Elapsed ms       :       3434       3950
---
Working on /data/zettar/zx/src/sweep/1x1024GiB with 56 threads...
NOTE: File size is not a multiple of block size times number of I/O threads, so the I/O threads write different amounts of data.
OPERATION RESULT TYPE        FIRST DONE  LAST DONE
========= ================   ==========  =========
DROPCACHE Elapsed ms       :       3373       3893
---
WRITE     Elapsed ms       :      90921     110448
          IOPS             :        589        593
          Throughput MiB/s :       9428       9493
          Total MiB        :     857280    1048576
---
DROPCACHE Elapsed ms       :       3772       4170
---
RMFILES   Elapsed ms       :          0          1
---
DROPCACHE Elapsed ms       :       3560       3961
---
===> /usr/local/sbin/mtelbencho.sh all done :)
Total test time: 53m:16s
===> Sweeps done. Extracting results...
===> Results extracted. Plotting...
===> /usr/local/sbin/graph_sweep.sh all done :)
Total test time: 2h:40m:35s

