Hello,
I am facing an issue with a filesystem (/dev/sda3); I see space used on it (around 365GB) when I am looking at the host with "df -h" command.
Code:
[root@srv_omega /]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sda3 443G 365G 56G 87% /
tmpfs 95G 56K 95G 1% /dev/shm
/dev/sda1 484M 39M 421M 9% /boot
/dev/sdb1 3.6T 1.3T 2.2T 36% /hadoop/disk1
/dev/sdc1 3.6T 1.3T 2.2T 37% /hadoop/disk2
/dev/sdd1 3.6T 1.3T 2.2T 36% /hadoop/disk3
/dev/sde1 3.6T 1.3T 2.2T 37% /hadoop/disk4
/dev/sdf1 3.6T 1.3T 2.2T 36% /hadoop/disk5
/dev/sdg1 3.6T 1.3T 2.2T 36% /hadoop/disk6
/dev/sdh1 3.6T 1.3T 2.2T 36% /hadoop/disk7
/dev/sdi1 3.6T 1.3T 2.2T 36% /hadoop/disk8
/dev/sdj1 3.6T 1.3T 2.2T 36% /hadoop/disk9
/dev/sdk1 3.6T 1.3T 2.2T 36% /hadoop/disk10
/dev/sdl1 3.6T 1.2T 2.3T 36% /hadoop/disk11
/dev/sdm1 3.6T 1.3T 2.2T 36% /hadoop/disk12
/dev/sdn1 3.6T 1.3T 2.2T 36% /hadoop/disk13
/dev/sdo1 3.6T 1.3T 2.2T 37% /hadoop/disk14
/dev/sdp1 3.6T 1.1T 2.4T 30% /hadoop/disk15
cm_processes 95G 8.2M 95G 1% /var/run/cloudera-scm-agent/process
I have looked if any hidden file might cause the issue, no joy.
Code:
[root@srv_omega /]# pwd
/
[root@srv_omega /]# ls -lrtha
total 121K
drwxr-xr-x 2 root root 4.0K Jun 28 2011 srv
drwxr-xr-x 2 root root 4.0K Jun 28 2011 mnt
drwxr-xr-x 2 root root 4.0K Jun 28 2011 media
drwxr-xr-x 2 root root 4.0K Dec 20 2012 cgroup
drwx------ 2 root root 16K Jun 2 2014 lost+found
drwxr-xr-x 2 root root 4.0K Jun 2 2014 selinux
-rw-r--r-- 1 root root 0 Jun 3 2014 .autorelabel
drwxr-xr-x 18 root root 4.0K Jun 5 2014 hadoop
drwxr-xr-x 21 root root 4.0K Jun 5 2014 var
dr-xr-xr-x 9 root root 12K Jun 20 2014 lib64
dr-xr-xr-x 2 root root 12K Jun 21 2014 sbin
dr-xr-xr-x 2 root root 4.0K Jun 21 2014 bin
dr-xr-xr-x 5 root root 1.0K Jun 22 2014 boot
dr-xr-x--- 5 root root 4.0K Jun 22 2014 root
drwxr-xr-x 6 root root 4.0K Jun 22 2014 opt
drwxr-xr-x 3 root root 4.0K Dec 10 19:11 home
dr-xr-xr-x 13 root root 4.0K Dec 12 16:18 lib
dr-xr-xr-x 1140 root root 0 Apr 30 15:11 proc
drwxr-xr-x 13 root root 0 Apr 30 15:11 sys
-rw-r--r-- 1 root root 0 Apr 30 15:11 .autofsck
drwxr-xr-x 2 root root 0 Apr 30 15:11 misc
drwxr-xr-x 2 root root 0 Apr 30 15:11 net
drwxr-xr-x 15 root root 4.0K Apr 30 15:12 usr
drwxr-xr-x 19 root root 4.6K Apr 30 15:12 dev
dr-xr-xr-x 27 root root 4.0K Apr 30 15:12 ..
dr-xr-xr-x 27 root root 4.0K Apr 30 15:12 .
drwxr-xr-x 122 root root 12K May 4 03:33 etc
drwxrwxrwt 16 root root 4.0K May 7 06:14 tmp
So I try to find where the space is used with a "du -sh" command
Code:
[root@srv_omega /]# pwd
/
[root@srv_omega /]# du -sh *
7.8M bin
29M boot
4.0K cgroup
280K dev
26M etc
19T hadoop
124K home
144M lib
26M lib64
16K lost+found
4.0K media
0 misc
4.0K mnt
0 net
7.9G opt
du: cannot access `proc/9170/task/27326/fdinfo/538': No such file or directory
du: cannot access `proc/45119/task/45119/fd/4': No such file or directory
du: cannot access `proc/45119/task/45119/fdinfo/4': No such file or directory
du: cannot access `proc/45119/fd/4': No such file or directory
du: cannot access `proc/45119/fdinfo/4': No such file or directory
du: cannot access `proc/45160': No such file or directory
0 proc
3.8M root
17M sbin
4.0K selinux
4.0K srv
0 sys
3.9M tmp
2.6G usr
16G var
So as far as I understand, only /hadoop is a suitable suspect (as cumulative size of all the other folders on "/" are well below the 365GB)
Code:
[root@srv_omega hadoop]# cd /
[root@srv_omega /]# cd /hadoop
[root@srv_omega hadoop]# ls -lrtha
total 72K
drwxr-xr-x 2 root root 4.0K Jun 5 2014 disk16
drwxr-xr-x 18 root root 4.0K Jun 5 2014 .
drwxr-xr-x 4 root root 4.0K Jun 22 2014 disk1
drwxr-xr-x 4 root root 4.0K Jun 22 2014 disk11
drwxr-xr-x 4 root root 4.0K Jun 22 2014 disk10
drwxr-xr-x 4 root root 4.0K Jun 22 2014 disk13
drwxr-xr-x 4 root root 4.0K Jun 22 2014 disk12
drwxr-xr-x 4 root root 4.0K Jun 22 2014 disk14
drwxr-xr-x 4 root root 4.0K Jun 22 2014 disk2
drwxr-xr-x 4 root root 4.0K Jun 22 2014 disk4
drwxr-xr-x 4 root root 4.0K Jun 22 2014 disk3
drwxr-xr-x 4 root root 4.0K Jun 22 2014 disk6
drwxr-xr-x 4 root root 4.0K Jun 22 2014 disk5
drwxr-xr-x 4 root root 4.0K Jun 22 2014 disk8
drwxr-xr-x 4 root root 4.0K Jun 22 2014 disk7
drwxr-xr-x 4 root root 4.0K Jun 22 2014 disk9
drwxr-xr-x 5 root root 4.0K Nov 19 20:02 disk15
dr-xr-xr-x 27 root root 4.0K Apr 30 15:12 ..
All folders from 1 to 15 are on different filesystems, so the folder disk16 seems to be the only option but there is nothing in it.
Code:
[root@srv_omega hadoop]# cd disk16/
[root@srv_omega disk16]# ls -lrtha
total 8.0K
drwxr-xr-x 18 root root 4.0K Jun 5 2014 ..
drwxr-xr-x 2 root root 4.0K Jun 5 2014 .
[root@srv_omega disk16]#
I just don't get it; no folder seems responsible for the "365Gb"...
Any idea on how I could try to find out where those "365GB" are ?