현재 ZFS 볼륨으로 만든 스토리지의 속도를 체크 해봤습니다.
약 50% 정도의 데이타가 들어 있는 시스템 입니다.
[root@BACKUP ~]# zpool status
pool: DATA
state: ONLINE
scan: scrub repaired 0 in 74h7m with 0 errors on Sun Dec 18 13:33:10 2016
config:
NAME STATE READ WRITE CKSUM
DATA ONLINE 0 0 0
raidz2-0 ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCJZAU9X ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCK4PB6X ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCJWMYSS ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCJUUDEX ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCK4HSKS ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCK2VXUS ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCK4M1RS ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCK5AP8X ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCK3UAUS ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCJYHW3S ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCK5WK1X ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCK1E43S ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK2338P4H7UPVC ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK2338P4H6ZKWC ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCK36N4X ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCK4K6YS ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK2381PCKYXZNB ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCJWPAYS ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCK2VK8S ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCJVDL4S ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCK5A1PX ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCJWP9BS ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCJV5DWS ONLINE 0 0 0
ata-HGST_HDN724040ALE640_PK1334PCJWSPJS ONLINE 0 0 0
errors: No known data errors
[root@BACKUP ~]#
[root@BACKUP ~]# zpool list
NAME SIZE ALLOC FREE EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT
DATA 87T 43.3T 43.7T - 14% 49% 1.00x ONLINE -
[root@BACKUP ~]#
[root@BACKUP ~]# sync && dd if=/dev/zero of=/DATA/dd_test bs=1k count=1000000 && sync
1000000+0 records in
1000000+0 records out
1024000000 bytes (1.0 GB) copied, 4.99862 s, 205 MB/s
[root@BACKUP ~]#
[root@BACKUP ~]# sync && dd if=/dev/zero of=/DATA/dd_test bs=1k count=10000000 && sync
10000000+0 records in
10000000+0 records out
10240000000 bytes (10 GB) copied, 48.8692 s, 210 MB/s
[root@BACKUP ~]#
[root@BACKUP ~]# sync && dd if=/dev/zero of=/DATA/dd_test bs=4k count=1000000 && sync
1000000+0 records in
1000000+0 records out
4096000000 bytes (4.1 GB) copied, 5.63363 s, 727 MB/s
[root@BACKUP ~]#
[root@BACKUP ~]# sync && dd if=/dev/zero of=/DATA/dd_test bs=8k count=1000000 && sync
1000000+0 records in
1000000+0 records out
8192000000 bytes (8.2 GB) copied, 6.59761 s, 1.2 GB/s
[root@BACKUP ~]#
[root@BACKUP ~]# sync && dd if=/dev/zero of=/DATA/dd_test bs=16k count=1000000 && sync
1000000+0 records in
1000000+0 records out
16384000000 bytes (16 GB) copied, 8.70386 s, 1.9 GB/s
[root@BACKUP ~]#
[root@BACKUP ~]# sync && dd if=/dev/zero of=/DATA/dd_test bs=256k count=100000 && sync
100000+0 records in
100000+0 records out
26214400000 bytes (26 GB) copied, 6.15736 s, 4.3 GB/s
[root@BACKUP ~]#
[root@BACKUP ~]# sync && dd if=/dev/zero of=/DATA/dd_test bs=512k count=100000 && sync
100000+0 records in
100000+0 records out
52428800000 bytes (52 GB) copied, 12.3036 s, 4.3 GB/s
[root@BACKUP ~]#
[root@BACKUP ~]# sync && dd if=/dev/zero of=/DATA/dd_test bs=1024k count=100000 && sync
100000+0 records in
100000+0 records out
104857600000 bytes (105 GB) copied, 24.5744 s, 4.3 GB/s
[root@BACKUP ~]#
안 나오더라구요. 그런데 이 테스트에서는 엄청 잘 나오는군요.
사용하신 명령어중에...
sync && dd if=/dev/zero of=/DATA/dd_test bs=1k count=1000000 && sync
... 요게 없는 파일을 임의로 만들어서 테스트 하는 건가요?
맞다면 저도 이 명령어로 내부 테스트를 좀 해봐야 할 것 같습니다.
CentOS6 에 ZFS 올린거고요.
mpt3sas_cm0: sending message unit reset !!
mpt3sas_cm0: message unit reset: SUCCESS
Refined TSC clocksource calibration: 3499.997 MHz.
Switching to clocksource tsc
mpt3sas_cm0: Allocated physical memory: size(17971 kB)
mpt3sas_cm0: Current Controller Queue Depth(9979),Max Controller Queue Depth(10240)
mpt3sas_cm0: Scatter Gather Elements per IO(128)
mpt3sas_cm0: LSISAS3008: FWVersion(10.00.00.00), ChipRevision(0x02), BiosVersion(08.25.00.00)
mpt3sas_cm0: Protocol=(
Initiator
,Target
),
Capabilities=(
TLR
,EEDP
,Snapshot Buffer
,Diag Trace Buffer
,Task Set Full
,NCQ
)
scsi0 : Fusion MPT SAS Host
mpt3sas_cm0: sending port enable !!
mpt3sas_cm0: host_add: handle(0x0001), sas_addr(0x5003048016a8b500), phys(8)
mpt3sas_cm0: expander_add: handle(0x0009), parent(0x0001), sas_addr(0x50030480178130bf), phys(38)
mpt3sas_cm0: port enable: SUCCESS
dd 명령어로 파일 쓰기하는겁니다.
순차쓰기 ... 랜덤 쓰기는 다시 올려 드릴께요. ㅎ
시간나는대로 테스트 해봐야겠습니다. ^0^)/
ZFS 옵션을 몇가지 쓰긴하는데..
ZFS ARC CACHE Min 1G / Max 8G Limit
compressratio 2.37x
recordsize 128K
compression on
sync disabled
autoexpand on
autoexpand on
[root@BACKUP DATA]# sync && dd if=/dev/zero of=/DATA/dd_test bs=16k count=1000000 && sync
1000000+0 records in
1000000+0 records out
16384000000 bytes (16 GB) copied, 8.65717 s, 1.9 GB/s
[root@BACKUP DATA]#
[root@BACKUP DATA]# sync && dd if=/dev/zero of=/DATA/dd_test bs=32k count=1000000 && sync
1000000+0 records in
1000000+0 records out
32768000000 bytes (33 GB) copied, 12.6994 s, 2.6 GB/s
[root@BACKUP DATA]#
[root@BACKUP DATA]# sync && dd if=/dev/zero of=/DATA/dd_test bs=64k count=1000000 && sync
1000000+0 records in
1000000+0 records out
65536000000 bytes (66 GB) copied, 20.6471 s, 3.2 GB/s
[root@BACKUP DATA]#
[root@BACKUP DATA]# sync && dd if=/dev/zero of=/DATA/dd_test bs=128k count=1000000 && sync
1000000+0 records in
1000000+0 records out
131072000000 bytes (131 GB) copied, 28.5942 s, 4.6 GB/s
[root@BACKUP DATA]#
[root@BACKUP DATA]# sync && dd if=/dev/zero of=/DATA/dd_test bs=256k count=1000000 && sync
1000000+0 records in
1000000+0 records out
262144000000 bytes (262 GB) copied, 59.8449 s, 4.4 GB/s
[root@BACKUP DATA]#
[root@BACKUP DATA]# sync && dd if=/dev/zero of=/DATA/dd_test bs=512k count=1000000 && sync
1000000+0 records in
1000000+0 records out
524288000000 bytes (524 GB) copied, 119.561 s, 4.4 GB/s
[root@BACKUP DATA]#
동일한 방식으로 DD를 10개, 20개, 50개, 100개, 1000개를 동시에 수행시켜보시면 좀더 현실적인 테스트를 하실 수 있을겁니다.
아시겠지만 ZFS는 DBMS를 올리기에 적당한 파일시스템은 아닙니다. 만약에 DB용 스토리지 테스트를 하실 수 있으시면 놀라실 겁니다.
그리고, 마지막에 sync는 필요없습니다. 보시는 결과/시간은 "dd"에 대한 결과이지 마지막 "sync" 의 시간이 포함되어 있지 않습니다.
그렇기 때문에 많으 데이터가 여전히 메모리에 있을겁니다. 아시겠지만 ZFS가 cacheing 무지 잘합니다.
마지막에 "SYNC"를 넣으신 이유가 모든 데이터가 DISK까지 들어가는 시간을 확인하시는 거라면 "time"을 DD와 SYNC를 모두 넣는 문장으로 바꾸셔야 합니다.
ZFS는 COW 파일시스템이기 때문에 영향이 적을 순 있겠습니다.
ARC 캐쉬는 MAX 8G 입니다.
ARC Summary: (HEALTHY)
Memory Throttle Count: 0
ARC Misc:
Deleted: 11.76b
Mutex Misses: 31.54k
Evict Skips: 31.54k
ARC Size: 12.68% 1.01 GiB
Target Size: (Adaptive) 26.20% 2.10 GiB
Min Size (Hard Limit): 12.50% 1.00 GiB
Max Size (High Water): 8:1 8.00 GiB
ARC Size Breakdown:
Recently Used Cache Size: 97.45% 2.04 GiB
Frequently Used Cache Size: 2.55% 54.71 MiB
ARC Hash Breakdown:
Elements Max: 610.25k
Elements Current: 10.25% 62.58k
Collisions: 2.04b
Chain Max: 8
Chains: 925
ARC Total accesses: 3.16b
Cache Hit Ratio: 57.50% 1.82b
Cache Miss Ratio: 42.50% 1.34b
Actual Hit Ratio: 57.50% 1.82b
Data Demand Efficiency: 57.99% 357.59m
[root@BACKUP ~]# sync && dd if=/dev/zero of=/DATA/dd_test2 bs=128k count=1000000 oflag=sync
1000000+0 records in
1000000+0 records out
131072000000 bytes (131 GB) copied, 422.089 s, 311 MB/s
[root@BACKUP ~]#
[root@BACKUP ~]# sync && dd if=/dev/zero of=/DATA/dd_test4 bs=128k count=1000000 oflag=sync
1000000+0 records in
1000000+0 records out
131072000000 bytes (131 GB) copied, 458.118 s, 286 MB/s
[root@BACKUP ~]#
[root@BACKUP ~]# sync && dd if=/dev/zero of=/DATA/dd_test0 bs=128k count=1000000 oflag=sync
1000000+0 records in
1000000+0 records out
131072000000 bytes (131 GB) copied, 455.995 s, 287 MB/s
[root@BACKUP ~]#
약 300 MB/s 정도 나오네요.
정확한 변수는 기억이 없는데 대략 다음과 같습니다.
생성
DEPTH=3, WIDTH=100, FILES =1000, FILESIZE=16MB
RD는
FILEIO/FILESELECTION RANDOM
xfersize=16KB
THREAD=10,20,50,100,1000
READ:WRITE = 7:3 (각 THREAD가 READ 7, WRITE 3)
INTERVAL=100 (중요)
으로 합니다.
R/W는 생각하시는 사용자의 환경에 맞춰서 만드시면 되겠지요.
한번 만들어 놓으면 그냥 수시로 돌리면 됩니다.