Forums

Home / Forums

You need to log in to create posts and topics. Login · Register

Vlan configuration

Pages: 1 2 3 4 5

erazmus
40 Posts
March 12, 2018, 7:13 pm
Quote from admin on March 12, 2018, 7:09 pm

What is the status of the following OSDs, is any up ?

52, 56, 65

Those OSD's (no longer) exist.

root@ceph0:~# ceph osd tree

ID  CLASS WEIGHT   TYPE NAME               STATUS REWEIGHT PRI-AFF 

 -1       14.00378 root default                                    

-12        0.43259     host ceph-lm3dc2-00                         

 44        0.06180         osd.44              up  1.00000 1.00000 

 45        0.06180         osd.45              up  1.00000 1.00000 

 50        0.06180         osd.50              up  1.00000 1.00000 

 60        0.06180         osd.60              up  1.00000 1.00000 

 61        0.06180         osd.61              up  1.00000 1.00000 

 62        0.06180         osd.62              up  1.00000 1.00000 

 63        0.06180         osd.63              up  1.00000 1.00000 

-13        0.98337     host ceph-lm3dc2-01                         

 23        0.26779         osd.23              up  1.00000 1.00000 

 24        0.26779         osd.24              up  1.00000 1.00000 

 25        0.26779         osd.25              up  1.00000 1.00000 

 46        0.06180         osd.46              up  1.00000 1.00000 

 47        0.06180         osd.47              up  1.00000 1.00000 

 58        0.02820         osd.58              up  1.00000 1.00000 

 59        0.02820         osd.59              up  1.00000 1.00000 

-14        1.46255     host ceph-lm3dc2-02                         

 26        0.26779         osd.26              up  1.00000 1.00000 

 27        0.26779         osd.27              up  1.00000 1.00000 

 48        0.06180         osd.48              up  1.00000 1.00000 

 49        0.06180         osd.49              up  1.00000 1.00000 

 69        0.26779         osd.69              up  1.00000 1.00000 

 70        0.26779         osd.70              up  1.00000 1.00000 

 71        0.26779         osd.71              up  1.00000 1.00000 

 -3        1.81310     host ceph0                                  

 30        1.81310         osd.30              up        0 1.00000 

 -4        0.54520     host ceph1                                  

  8        0.54520         osd.8               up        0 1.00000 

-15        1.33896     host ceph10                                 

 51        0.26779         osd.51              up  1.00000 1.00000 

 53        0.26779         osd.53              up  1.00000 1.00000 

 54        0.26779         osd.54              up  1.00000 1.00000 

 55        0.26779         osd.55              up  1.00000 1.00000 

 57        0.26779         osd.57              up        0 1.00000 

 -2        1.49094     host ceph2                                  

  0        0.13339         osd.0               up  1.00000 1.00000 

  1        0.27269         osd.1               up  1.00000 1.00000 

  2        0.27269         osd.2               up  1.00000 1.00000 

  3        0.27269         osd.3               up  1.00000 1.00000 

  4        0.27269         osd.4               up  1.00000 1.00000 

  5        0.13339         osd.5               up  1.00000 1.00000 

  6        0.13339         osd.6               up  1.00000 1.00000 

 -5              0     host ceph3                                  

 -6        1.73524     host ceph4                                  

 16        0.12849         osd.16              up  1.00000 1.00000 

 17        0.26779         osd.17              up  1.00000 1.00000 

 18        0.26779         osd.18              up  1.00000 1.00000 

 19        0.26779         osd.19              up  1.00000 1.00000 

 20        0.26779         osd.20              up  1.00000 1.00000 

 21        0.26779         osd.21              up  1.00000 1.00000 

 22        0.26779         osd.22              up  1.00000 1.00000 

 -7        1.34184     host ceph5                                  

 64        0.35818         osd.64              up  1.00000 1.00000 

 66        0.26729         osd.66              up  1.00000 1.00000 

 67        0.35818         osd.67              up  1.00000 1.00000 

 68        0.35818         osd.68              up  1.00000 1.00000 

 -8        0.35818     host ceph6                                  

 28        0.35818         osd.28              up  1.00000 1.00000 

 -9        0.26729     host ceph7                                  

 29        0.26729         osd.29              up  1.00000 1.00000 

-10        1.66855     host ceph8                                  

  7        0.06180         osd.7               up  1.00000 1.00000 

 31        0.26779         osd.31              up  1.00000 1.00000 

 32        0.26779         osd.32              up        0 1.00000 

 33        0.26779         osd.33              up  1.00000 1.00000 

 34        0.26779         osd.34              up  1.00000 1.00000 

 35        0.26779         osd.35              up  1.00000 1.00000 

 36        0.26779         osd.36              up  1.00000 1.00000 

-11        0.56598     host ceph9                                  

 37        0.06180         osd.37              up  1.00000 1.00000 

 38        0.06180         osd.38              up  1.00000 1.00000 

 39        0.06180         osd.39              up  1.00000 1.00000 

 40        0.06180         osd.40              up  1.00000 1.00000 

 41        0.06180         osd.41              up  1.00000 1.00000 

 42        0.12849         osd.42              up  1.00000 1.00000 

 43        0.12849         osd.43              up  1.00000 1.00000 

root@ceph0:~# 

#31

admin
2,930 Posts
March 12, 2018, 7:44 pm

if you can do not create new OSDs from them.

#32

erazmus
40 Posts
March 12, 2018, 7:53 pm
Quote from admin on March 12, 2018, 7:44 pm

if you can do not create new OSDs from them.

Sorry, I'm unclear what you mean.

#33

admin
2,930 Posts
March 12, 2018, 8:01 pm

I just want that these 3 disks, not be re-added as new OSDs for now.

#34

erazmus
40 Posts
March 12, 2018, 8:23 pm
Quote from admin on March 12, 2018, 8:01 pm

I just want that these 3 disks, not be re-added as new OSDs for now.

Okay, I will not re-add them.

#35

admin
2,930 Posts
March 12, 2018, 10:23 pm

Can you try

ceph pg repair 1.e0e --cluster CLUSTER_NAME

For the deleted  OSD 52, 56, 65 disks,  can you try to mount their first partition in a temp dir such as

mount /dev/sdX1 /mnt

There are some suggestions to create a new OSD with same id(s)  as the deleted ones (52,56,65) and this may allow the cluster to proceed...I am trying to find some commands to create an OSD with specific id but did not find something straightforward so far.

The current assigned/acting  OSDs for 1.e0e are OSD # 2, 35, 23    can you see how much data they have for this PG:

du -ch /var/lib/ceph/osd/CLUSTER_NAME-OSD_ID/current/1.e0e_head/
such as:
du -ch /var/lib/ceph/osd/demo-2/current/1.e0e_head/

Last edited on March 13, 2018, 12:10 am by admin · #36

erazmus
40 Posts
March 13, 2018, 4:05 am
Quote from admin on March 12, 2018, 10:23 pm

Can you try

ceph pg repair 1.e0e --cluster CLUSTER_NAME

Didn't change the status at all.

For the deleted  OSD 52, 56, 65 disks,  can you try to mount their first partition in a temp dir such as

mount /dev/sdX1 /mnt

There are some suggestions to create a new OSD with same id(s)  as the deleted ones (52,56,65) and this may allow the cluster to proceed...I am trying to find some commands to create an OSD with specific id but did not find something straightforward so far.

OSD52's drive and OSD56's drive both have a blank GPT partition table so no partition to mount - perhaps a failed upgrade of the OSD?

OSD65 has a good partition table, and I can mount the partition to a temp mount point and I have OSD-like files in there.

The current assigned/acting  OSDs for 1.e0e are OSD # 2, 35, 23    can you see how much data they have for this PG:

du -ch /var/lib/ceph/osd/CLUSTER_NAME-OSD_ID/current/1.e0e_head/
such as:
du -ch /var/lib/ceph/osd/demo-2/current/1.e0e_head/

OSD2 is on a monitor node, and there's no 'current' directory there - maybe because the OSD has been upgraded?

OSD35 has 0 bytes in 1.e0e_head

OSD 23 doesn't have a 1.e0e_head file in the current directory.

#37

admin
2,930 Posts
March 13, 2018, 8:19 am

OSDs 52, 56 were probably among those deleted.

Can you please check if 65 disk has stored data for PG 1.e0e via same command ?

Can you please check if any other running OSD  has stored data for PG 1.e0e ?

#38

erazmus
40 Posts
March 13, 2018, 5:24 pm
Quote from admin on March 13, 2018, 8:19 am

OSDs 52, 56 were probably among those deleted.

Can you please check if 65 disk has stored data for PG 1.e0e via same command ?

Looks line not an OSD. Here's a directory:

root@ceph5:/mnt# ls -al

total 20

drwxr-xr-x  2 ceph ceph   83 Mar 11 10:09 .

drwxr-xr-x 23 root root 4096 Dec 14 11:42 ..

-rw-r--r--  1 ceph ceph   37 Mar 11 10:09 ceph_fsid

-rw-r--r--  1 ceph ceph   37 Mar 11 10:09 fsid

lrwxrwxrwx  1 ceph ceph   58 Mar 11 10:09 journal -> /dev/disk/by-partuuid/5ccb9f57-66fc-4545-83f4-003406f6cad9

-rw-r--r--  1 ceph ceph   37 Mar 11 10:09 journal_uuid

-rw-r--r--  1 ceph ceph   21 Mar 11 10:09 magic

root@ceph5:/mnt#

Can you please check if any other running OSD  has stored data for PG 1.e0e ?

osd.66 (on host ceph5)

osd.32 (on host ceph8)

osd.35 (on host ceph8)

osd.57 (on host ceph10)

 

#39

admin
2,930 Posts
March 13, 2018, 6:08 pm

Hi,

Can you run the du command to see how much data these OSDs have for the problem PG.

#40
Pages: 1 2 3 4 5