hwmon/pmbus: sensor fluctuations and out-of-range values of power supplies

[jktjkt]

sdn-bidi-cplus1572-PGCL250304 ~ # velia-list-alarms 
   Resource              Severity  Detail                                                    Timestamp                            Status
⏸   velia-system.service  critical  systemd unit state: (failed, failed-before-auto-restart)  2025-05-23T08:51:14.332132841+00:00  cleared
                         cleared   systemd unit state: (activating, auto-restart-queued)     2025-05-23T08:51:24.471437497+00:00  
                         critical  systemd unit state: (activating, auto-restart)            2025-05-23T08:51:14.388608352+00:00  
                         cleared   systemd unit state: (activating, auto-restart-queued)     2025-05-23T08:51:06.568808888+00:00  
                         critical  systemd unit state: (activating, auto-restart)            2025-05-23T08:51:06.516304191+00:00  
⏷   ne:psu1:voltage-in    cleared   Sensor value is within normal parameters.                 2025-05-23T13:56:27.311500830+00:00  cleared
                         cleared   Sensor value is within normal parameters.                 2025-05-23T14:02:10.625083914+00:00  
                         critical  Sensor value crossed low threshold (2000 < 90000).        2025-05-23T14:02:08.654824534+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T14:01:07.550300342+00:00  
                         critical  Sensor value crossed low threshold (2000 < 90000).        2025-05-23T14:01:05.578179118+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T14:01:01.680226415+00:00  
                         critical  Sensor value crossed low threshold (3250 < 90000).        2025-05-23T14:00:59.680230614+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T14:00:07.948797111+00:00  
                         critical  Sensor value crossed low threshold (54500 < 90000).       2025-05-23T14:00:05.958319641+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:59:08.588271401+00:00  
                         critical  Sensor value crossed low threshold (3250 < 90000).        2025-05-23T13:59:06.630017712+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:58:30.200526699+00:00  
                         critical  Sensor value crossed low threshold (10750 < 90000).       2025-05-23T13:58:28.223341590+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:58:24.343886480+00:00  
                         critical  Sensor value crossed low threshold (2000 < 90000).        2025-05-23T13:58:22.366443879+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:57:55.551575914+00:00  
                         critical  Sensor value crossed low threshold (3250 < 90000).        2025-05-23T13:57:53.557831863+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:57:05.719047437+00:00  
                         critical  Sensor value crossed low threshold (3250 < 90000).        2025-05-23T13:57:03.673390741+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:56:50.272905417+00:00  
                         critical  Sensor value crossed low threshold (10750 < 90000).       2025-05-23T13:56:46.410945779+00:00  
                         critical  Sensor value crossed low threshold (2000 < 90000).        2025-05-23T13:56:25.326097069+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:56:08.068691819+00:00  
                         critical  Sensor value crossed low threshold (2000 < 90000).        2025-05-23T13:56:06.086121811+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:56:02.225939945+00:00  
                         critical  Sensor value crossed low threshold (10750 < 90000).       2025-05-23T13:56:00.243734669+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:55:25.794328130+00:00  
                         critical  Sensor value crossed low threshold (3250 < 90000).        2025-05-23T13:55:23.819317371+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:54:45.559829430+00:00  
                         critical  Sensor value crossed low threshold (10750 < 90000).       2025-05-23T13:54:43.572751305+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:53:48.087960818+00:00  
                         critical  Sensor value crossed low threshold (10750 < 90000).       2025-05-23T13:53:46.113620358+00:00  
⏶   ne:psu1:voltage-in    cleared   Sensor value is within normal parameters.                 2025-05-23T12:56:10.453958495+00:00  cleared
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:57:07.692066774+00:00  
                         critical  Sensor value crossed high threshold (316000 > 264000).    2025-05-23T13:57:05.648598148+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:51:43.292553626+00:00  
                         critical  Sensor value crossed high threshold (287000 > 264000).    2025-05-23T13:51:41.323585970+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:36:03.336330463+00:00  
                         critical  Sensor value crossed high threshold (418000 > 264000).    2025-05-23T13:36:01.344319884+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:32:26.374672056+00:00  
                         critical  Sensor value crossed high threshold (331500 > 264000).    2025-05-23T13:32:24.404210739+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:17:27.408230685+00:00  
                         critical  Sensor value crossed high threshold (291500 > 264000).    2025-05-23T13:17:25.413994965+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:15:54.924287686+00:00  
                         critical  Sensor value crossed high threshold (312000 > 264000).    2025-05-23T13:15:52.817587156+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:14:35.836326019+00:00  
                         critical  Sensor value crossed high threshold (281500 > 264000).    2025-05-23T13:14:33.866230071+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:14:28.042457019+00:00  
                         critical  Sensor value crossed high threshold (294000 > 264000).    2025-05-23T13:14:26.063232643+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:12:42.866008225+00:00  
                         critical  Sensor value crossed high threshold (407500 > 264000).    2025-05-23T13:12:40.893074783+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T13:11:45.260004902+00:00  
                         critical  Sensor value crossed high threshold (286500 > 264000).    2025-05-23T13:11:43.276660506+00:00  
                         critical  Sensor value crossed high threshold (311000 > 264000).    2025-05-23T12:56:08.480110993+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T12:55:32.012536288+00:00  
                         critical  Sensor value crossed high threshold (296500 > 264000).    2025-05-23T12:55:30.031339891+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T12:53:16.358630216+00:00  
                         critical  Sensor value crossed high threshold (379000 > 264000).    2025-05-23T12:53:14.379680036+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T12:51:05.586671193+00:00  
                         critical  Sensor value crossed high threshold (282500 > 264000).    2025-05-23T12:51:03.612994058+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T12:43:51.255187811+00:00  
                         critical  Sensor value crossed high threshold (312000 > 264000).    2025-05-23T12:43:49.273794267+00:00  
                         cleared   Sensor value is within normal parameters.                 2025-05-23T12:40:56.780322108+00:00  
                         critical  Sensor value crossed high threshold (332500 > 264000).    2025-05-23T12:40:54.800962087+00:00  

This is on the most recent HW:

sdn-bidi-cplus1572-PGCL250304 ~ # velia-list-hardware 
HW                       Manufacturer  Model                                           Version                    S/N
ne                       PEI-Genesis   sdn-bidi-cplus1572-g2 (PG-CL-SDN_dualBiDi-C-L)  1                          PGCL250304
ne:ctrl                                                                                                           0910C30854100840403BA080A08000AA
ne:ctrl:carrier          SolidRun      Clearfog Base (SRCFCBE000CV14)                  1.4                        IP01195231800006
ne:ctrl:carrier:console                                                                                           DQ00G4UC
ne:ctrl:carrier:eeprom                                                                                            294100B1385A
ne:ctrl:emmc                           8GTF4R                                                                     0x35c55f91
ne:ctrl:som              SolidRun      A38x SOM (SRM6828S32D01GE008V21C0)              2.1                        IP01195231800006
ne:ctrl:som:eeprom                                                                                                80342872AC07
ne:edfa-c-band                         FIXED GAIN                                      1.1.0                      346733
ne:edfa-narrow-1572                    FIXED GAIN                                      1.1.0                      346691
ne:fans                                                                                                           0910C30854100842000DA080A08000A5
ne:pdu                   3Y POWER      YH-5151E (URP1X151AH)                           B01R P2J700A01 A20         SA200T302229001114
ne:psu1                  3Y POWER      YM-2151E (URP1X151AM)                           B01R        P2J700A04 A17  SA170T292229004149 
ne:psu2                  3Y POWER      YM-2151E (URP1X151AM)                           B01R        P2J700A04 A17  SA170T292229004150

[jktjkt]

So, with I2C tracing enabled:

# echo 1 > /sys/kernel/debug/tracing/events/i2c/enable
# echo 1 > /sys/kernel/debug/tracing/events/smbus/enable
# echo adapter_nr==2 > /sys/kernel/debug/tracing/events/i2c/filter
# echo adapter_nr==2 > /sys/kernel/debug/tracing/events/smbus/filter
# echo 1 > /sys/kernel/debug/tracing/tracing_on
# tail -f /sys/kernel/debug/tracing/trace

I managed to reproduce some flaky reads from a PSU, like this. This one reads the 230V input (it reports that there's no voltage there, but that's actually wrong, it is plugged in):

sdn-bidi-cplus1572-PGCL250304 ~ # cat /sys/class/hwmon/hwmon8/in1_input 
0

...and the trace is:

cat-2671    [001] .....  1779.651696: smbus_write: i2c-2 a=058 f=0004 c=0 BYTE_DATA l=1 [00]
cat-2671    [001] .....  1779.651702: i2c_write: i2c-2 #0 a=058 f=0004 l=3 [00-00-ea]
cat-2671    [001] .....  1779.652361: i2c_result: i2c-2 n=1 ret=1
cat-2671    [001] .....  1779.652364: smbus_result: i2c-2 a=058 f=0004 c=0 BYTE_DATA wr res=0
cat-2671    [001] .....  1779.712145: smbus_read: i2c-2 a=058 f=0004 c=88 WORD_DATA
cat-2671    [001] .....  1779.712149: i2c_write: i2c-2 #0 a=058 f=0004 l=1 [88]
cat-2671    [001] .....  1779.712151: i2c_read: i2c-2 #1 a=058 f=0005 l=3
cat-2671    [001] .....  1779.712911: i2c_reply: i2c-2 #1 a=058 f=0005 l=3 [00-f0-1f]
cat-2671    [001] .....  1779.712912: i2c_result: i2c-2 n=2 ret=2
cat-2671    [001] .....  1779.712915: smbus_reply: i2c-2 a=058 f=0004 c=88 WORD_DATA l=2 [00-f0]
cat-2671    [001] .....  1779.712916: smbus_result: i2c-2 a=058 f=0004 c=88 WORD_DATA rd res=0

Reading the 12V output yields a number that's out of range:

sdn-bidi-cplus1572-PGCL250304 ~ # cat /sys/class/hwmon/hwmon8/in2_input 
6118000

...with the trace:

cat-2672    [001] .....  1788.012572: smbus_read: i2c-2 a=058 f=0004 c=8b WORD_DATA
cat-2672    [001] .....  1788.012578: i2c_write: i2c-2 #0 a=058 f=0004 l=1 [8b]
cat-2672    [001] .....  1788.012580: i2c_read: i2c-2 #1 a=058 f=0005 l=3
cat-2672    [001] .....  1788.013343: i2c_reply: i2c-2 #1 a=058 f=0005 l=3 [e6-17-a3]
cat-2672    [001] .....  1788.013345: i2c_result: i2c-2 n=2 ret=2
cat-2672    [001] .....  1788.013347: smbus_reply: i2c-2 a=058 f=0004 c=8b WORD_DATA l=2 [e6-17]
cat-2672    [001] .....  1788.013349: smbus_result: i2c-2 a=058 f=0004 c=8b WORD_DATA rd res=0

And finally, the 5Vsb:

sdn-bidi-cplus1572-PGCL250304 ~ # cat /sys/class/hwmon/hwmon8/in3_input 
5035

trace:

cat-2673    [001] .....  1796.071797: smbus_write: i2c-2 a=058 f=0004 c=0 BYTE_DATA l=1 [20]
cat-2673    [001] .....  1796.071803: i2c_write: i2c-2 #0 a=058 f=0004 l=3 [00-20-0a]
cat-2673    [001] .....  1796.072470: i2c_result: i2c-2 n=1 ret=1
cat-2673    [001] .....  1796.072472: smbus_result: i2c-2 a=058 f=0004 c=0 BYTE_DATA wr res=0
cat-2673    [000] .....  1796.131144: smbus_read: i2c-2 a=058 f=0004 c=8b WORD_DATA
cat-2673    [000] .....  1796.131151: i2c_write: i2c-2 #0 a=058 f=0004 l=1 [8b]
cat-2673    [000] .....  1796.131152: i2c_read: i2c-2 #1 a=058 f=0005 l=3
cat-2673    [000] .....  1796.131916: i2c_reply: i2c-2 #1 a=058 f=0005 l=3 [12-0a-b0]
cat-2673    [000] .....  1796.131918: i2c_result: i2c-2 n=2 ret=2
cat-2673    [000] .....  1796.131921: smbus_reply: i2c-2 a=058 f=0004 c=8b WORD_DATA l=2 [12-0a]
cat-2673    [000] .....  1796.131923: smbus_result: i2c-2 a=058 f=0004 c=8b WORD_DATA rd res=0

Before that, I checked this output of a better tool:

sdn-bidi-cplus1572-PGCL250304 ~ # pmbus_peek -b /dev/i2c-2 -p -v -s -g 0x00 -f 0x58
PMBus slave on /dev/i2c-2, address 0x58

Inventory Data:
  Manufacturer:         3Y POWER
  Model:                YM-2151E
  Revision:             A
  IC Device:            PIC24FJ32GA004

PMBus revisions (0x22): part I, ver 1.1; part II, ver 1.2
Capabilities (0x90):    PEC, SMBALERT#, 100 KHz

Application Profiles:
 Server AC-DC Power Supply: rev 0.2

Status 0000: 

Attribute Values:
  page                  00: (BITMAP)
  operation             00: (BITMAP)
  on_off_config         15: (BITMAP)
  vout_mode             17: (BITMAP)
  fan_config_1_2        90: (BITMAP)
  fan_command_1         0000: 0
  fan_command_2         0000: 0
  iout_oc_warn_limit    0010: 16 Amperes
  toff_fall             0000: 0 milliseconds
  ein                   dc0700ad3b02: 2012
  eout                  d50c1fac3b02: 1.01906e+06
  vin                   f000: 0 Volts
  iin                   c000: 0 Amperes
  vout                  17e6: 11.9492 Volts
  iout                  c0a8: 0.65625 Amperes
  temperature_1         0020: 32 degrees Celsius
  temperature_2         0022: 34 degrees Celsius
  temperature_3         0000: 0 degrees Celsius
  fan_speed_1           1a19: 4296
  fan_speed_2           0000: 0
  frequency             f0c7: 49.75
  pout                  f020: 8 Watts
  pin                   f000: 0 Watts
  mfr_vin_min           005a: 90 Volts
  mfr_vin_max           0108: 264 Volts
  mfr_iin_max           0003: 3 Amperes
  mfr_pin_max           00c8: 200 Watts
  mfr_vout_min          1748: -3360 Volts
  mfr_iout_max          18b0: 1408 Amperes
  mfr_pout_max          0096: 150 Watts
  mfr_tambient_max      0032: 50 degrees Celsius
  mfr_tambient_min      0000: 0 degrees Celsius

and:

sdn-bidi-cplus1572-PGCL250304 ~ # pmbus_peek -b /dev/i2c-2 -p -v -s -g 0x20 -f 0x58
PMBus slave on /dev/i2c-2, address 0x58

Inventory Data:
  Manufacturer:         3Y POWER
  Model:                YM-2151E
  Revision:             A
  IC Device:            PIC24FJ32GA004

PMBus revisions (0x22): part I, ver 1.1; part II, ver 1.2
Capabilities (0x90):    PEC, SMBALERT#, 100 KHz

Application Profiles:
 Server AC-DC Power Supply: rev 0.2

Status 0000: 

Attribute Values:
  page                  20: (BITMAP)
  operation             00: (BITMAP)
  on_off_config         15: (BITMAP)
  vout_mode             17: (BITMAP)
  fan_config_1_2        90: (BITMAP)
  fan_command_1         0000: 0
  fan_command_2         0000: 0
  iout_oc_warn_limit    f807: 3.5 Amperes
  toff_fall             0000: 0 milliseconds
  ein                   dc07000e3f02: 2012
  eout                  49231f0f3f02: 1.02481e+06
  vin                   f000: 0 Volts
  iin                   c000: 0 Amperes
  vout                  0a12: 5.03516 Volts
  iout                  c018: 0.09375 Amperes
  temperature_1         0020: 32 degrees Celsius
  temperature_2         0022: 34 degrees Celsius
  temperature_3         0000: 0 degrees Celsius
  fan_speed_1           1a19: 4296
  fan_speed_2           0000: 0
  frequency             f0c7: 49.75
  pout                  f01c: 7 Watts
  pin                   f000: 0 Watts
  mfr_vin_min           005a: 90 Volts
  mfr_vin_max           0108: 264 Volts
  mfr_iin_max           0003: 3 Amperes
  mfr_pin_max           00c8: 200 Watts
  mfr_vout_min          0998: 816 Volts
  mfr_iout_max          0a64: 612 Amperes
  mfr_pout_max          0096: 150 Watts
  mfr_tambient_max      0032: 50 degrees Celsius
  mfr_tambient_min      0000: 0 degrees Celsius

Traces are attached: pmbus-traces.txt

[jktjkt]

After a lot of PSU wiggling (nope, simply deregistering and re-registering the PSU from the kernel side over and over again was not enough), here's what happens:

[ 4975.415256] fsp3y 2-0058: pmbus_check_status_cml: PB_CML_FAULT_INVALID_COMMAND in PMBUS_STATUS_CML: -EIO
[ 4975.415658] fsp3y 2-0058: pmbus_check_register(reg 0x20): failing due to rv 0xfffffffb
[ 4975.415666] fsp3y 2-0058: PMBUS_VOUT_MODE: pmbus_check_byte_register failed for page 0; using hex value of ffffffff
[ 4975.415672] fsp3y 2-0058: VOUT_MODE was out of range, so we aren't doing a thing
[ 4975.469043] fsp3y 2-0058: VOUT_MODE at page 1: hex 17
[ 4975.469057] fsp3y 2-0058: VOUT_MODE: linear, exponent=f7

...which means that the kernel, in pmbus_identify_common(), tries to read register 0x20 (VOUT_MODE), and when it checks the STATUS_WORD register (0x79), it has the CML bit set, which leads to reading the STATUS_CML (0x7e), which says "invalid or unsupported command received". The kernel code won't recover from that, and it will instead use the default value as an exponent (which I guess is zero)

Here's the full trace:

 veliad-hardware-8109    [000] .....  4975.402612: smbus_read: i2c-2 a=058 f=0000 c=9a BLOCK_DATA
 veliad-hardware-8109    [000] .....  4975.404105: smbus_reply: i2c-2 a=058 f=0000 c=9a BLOCK_DATA l=9 [08-59-4d-2d-32-31-35-31-45]
 veliad-hardware-8109    [000] .....  4975.404107: smbus_result: i2c-2 a=058 f=0000 c=9a BLOCK_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.404110: smbus_read: i2c-2 a=058 f=0000 c=0 BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.405471: smbus_reply: i2c-2 a=058 f=0000 c=0 BYTE_DATA l=1 [00]
 veliad-hardware-8109    [000] .....  4975.405473: smbus_result: i2c-2 a=058 f=0000 c=0 BYTE_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.405477: smbus_read: i2c-2 a=058 f=0000 c=20 BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.405996: smbus_reply: i2c-2 a=058 f=0000 c=20 BYTE_DATA l=1 [17]
 veliad-hardware-8109    [000] .....  4975.405997: smbus_result: i2c-2 a=058 f=0000 c=20 BYTE_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.406001: smbus_read: i2c-2 a=058 f=0000 c=19 BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.407225: smbus_reply: i2c-2 a=058 f=0000 c=19 BYTE_DATA l=1 [90]
 veliad-hardware-8109    [000] .....  4975.407228: smbus_result: i2c-2 a=058 f=0000 c=19 BYTE_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.407231: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [000] .....  4975.408102: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [40-08]
 veliad-hardware-8109    [000] .....  4975.408104: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.408106: smbus_read: i2c-2 a=058 f=0004 c=10 BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.408763: smbus_result: i2c-2 a=058 f=0004 c=10 BYTE_DATA rd res=-74
 veliad-hardware-8109    [000] .....  4975.408766: smbus_read: i2c-2 a=058 f=0004 c=98 BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.409399: smbus_reply: i2c-2 a=058 f=0004 c=98 BYTE_DATA l=1 [22]
 veliad-hardware-8109    [000] .....  4975.409400: smbus_result: i2c-2 a=058 f=0004 c=98 BYTE_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.409403: smbus_write: i2c-2 a=058 f=0004 c=0 BYTE_DATA l=1 [00]
 veliad-hardware-8109    [000] .....  4975.409915: smbus_result: i2c-2 a=058 f=0004 c=0 BYTE_DATA wr res=0
 veliad-hardware-8109    [000] .....  4975.409916: smbus_read: i2c-2 a=058 f=0004 c=0 BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.410549: smbus_reply: i2c-2 a=058 f=0004 c=0 BYTE_DATA l=1 [00]
 veliad-hardware-8109    [000] .....  4975.410549: smbus_result: i2c-2 a=058 f=0004 c=0 BYTE_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.410551: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.410936: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.410937: smbus_write: i2c-2 a=058 f=0004 c=0 BYTE_DATA l=1 [01]
 veliad-hardware-8109    [000] .....  4975.411447: smbus_result: i2c-2 a=058 f=0004 c=0 BYTE_DATA wr res=0
 veliad-hardware-8109    [000] .....  4975.411947: smbus_read: i2c-2 a=058 f=0004 c=0 BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.412579: smbus_reply: i2c-2 a=058 f=0004 c=0 BYTE_DATA l=1 [01]
 veliad-hardware-8109    [000] .....  4975.412580: smbus_result: i2c-2 a=058 f=0004 c=0 BYTE_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.412581: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.412966: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.412968: smbus_read: i2c-2 a=058 f=0004 c=20 BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.413862: smbus_reply: i2c-2 a=058 f=0004 c=20 BYTE_DATA l=1 [17]
 veliad-hardware-8109    [000] .....  4975.413863: smbus_result: i2c-2 a=058 f=0004 c=20 BYTE_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.413866: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [000] .....  4975.414619: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [42-08]
 veliad-hardware-8109    [000] .....  4975.414620: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.414621: smbus_read: i2c-2 a=058 f=0004 c=7e BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.415251: smbus_reply: i2c-2 a=058 f=0004 c=7e BYTE_DATA l=1 [80]
 veliad-hardware-8109    [000] .....  4975.415252: smbus_result: i2c-2 a=058 f=0004 c=7e BYTE_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.415271: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.415657: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.415677: smbus_write: i2c-2 a=058 f=0004 c=0 BYTE_DATA l=1 [20]
 veliad-hardware-8109    [000] .....  4975.416190: smbus_result: i2c-2 a=058 f=0004 c=0 BYTE_DATA wr res=0
 veliad-hardware-8109    [000] .....  4975.466619: smbus_read: i2c-2 a=058 f=0004 c=20 BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.467262: smbus_reply: i2c-2 a=058 f=0004 c=20 BYTE_DATA l=1 [17]
 veliad-hardware-8109    [000] .....  4975.467264: smbus_result: i2c-2 a=058 f=0004 c=20 BYTE_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.467266: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [000] .....  4975.468017: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [40-08]
 veliad-hardware-8109    [000] .....  4975.468018: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.468020: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.468405: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.468407: smbus_read: i2c-2 a=058 f=0004 c=20 BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.469039: smbus_reply: i2c-2 a=058 f=0004 c=20 BYTE_DATA l=1 [17]
 veliad-hardware-8109    [000] .....  4975.469040: smbus_result: i2c-2 a=058 f=0004 c=20 BYTE_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.469078: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.469465: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.469473: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.469859: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.469865: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.470250: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.470256: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.470642: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.470668: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.471053: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.471059: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.471445: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.471451: smbus_write: i2c-2 a=058 f=0004 c=0 BYTE_DATA l=1 [00]
 veliad-hardware-8109    [000] .....  4975.471964: smbus_result: i2c-2 a=058 f=0004 c=0 BYTE_DATA wr res=0
 veliad-hardware-8109    [001] .....  4975.527589: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [001] .....  4975.528356: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [40-08]
 veliad-hardware-8109    [001] .....  4975.528358: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [001] .....  4975.528362: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.528747: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.528763: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.529164: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.529186: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.529574: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.529583: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.529968: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.529977: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.530371: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.530402: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.530788: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.530795: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.531180: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.531189: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [001] .....  4975.531941: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [40-08]
 veliad-hardware-8109    [001] .....  4975.531942: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [001] .....  4975.531944: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.532328: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.532336: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.532721: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.532730: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.533115: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.533124: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.533508: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.533516: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.533902: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.533929: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.534315: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.534323: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.534708: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.534716: smbus_write: i2c-2 a=058 f=0004 c=0 BYTE_DATA l=1 [20]
 veliad-hardware-8109    [001] .....  4975.535229: smbus_result: i2c-2 a=058 f=0004 c=0 BYTE_DATA wr res=0
 veliad-hardware-8109    [001] .....  4975.586104: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [001] .....  4975.586867: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [00-00]
 veliad-hardware-8109    [001] .....  4975.586868: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [001] .....  4975.586872: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.587257: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.587268: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.587659: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.587680: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.588066: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.588094: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.588480: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.588487: smbus_write: i2c-2 a=058 f=0004 c=0 BYTE_DATA l=1 [00]
 veliad-hardware-8109    [001] .....  4975.589000: smbus_result: i2c-2 a=058 f=0004 c=0 BYTE_DATA wr res=0
 veliad-hardware-8109    [001] .....  4975.646093: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [001] .....  4975.646853: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [00-00]
 veliad-hardware-8109    [001] .....  4975.646855: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [001] .....  4975.646858: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.647243: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.647253: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.647639: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.647654: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.648041: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.648563: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.648949: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.648975: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.649361: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.649367: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [001] .....  4975.650119: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [00-00]
 veliad-hardware-8109    [001] .....  4975.650120: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [001] .....  4975.650121: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.650506: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.650510: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.650950: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.650964: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.651351: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.651360: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.651746: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.651773: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.652159: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.652167: smbus_write: i2c-2 a=058 f=0004 c=0 BYTE_DATA l=1 [20]
 veliad-hardware-8109    [001] .....  4975.652680: smbus_result: i2c-2 a=058 f=0004 c=0 BYTE_DATA wr res=0
 veliad-hardware-8109    [001] .....  4975.706626: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [001] .....  4975.707411: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [00-00]
 veliad-hardware-8109    [001] .....  4975.707414: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [001] .....  4975.707419: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.707805: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.707820: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.708206: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.708245: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.708632: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.708639: smbus_write: i2c-2 a=058 f=0004 c=0 BYTE_DATA l=1 [00]
 veliad-hardware-8109    [001] .....  4975.709151: smbus_result: i2c-2 a=058 f=0004 c=0 BYTE_DATA wr res=0
 veliad-hardware-8109    [001] .....  4975.766652: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [001] .....  4975.767414: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [00-00]
 veliad-hardware-8109    [001] .....  4975.767416: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [001] .....  4975.767420: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.767805: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.767817: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.768203: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.768220: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.768606: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.768614: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.768999: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.769026: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.769411: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.769484: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.769870: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.769878: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.770264: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.770271: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.770656: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.770663: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.771048: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.771074: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.771460: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.771467: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [001] .....  4975.772219: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [00-00]
 veliad-hardware-8109    [001] .....  4975.772220: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [001] .....  4975.772221: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.772606: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.772638: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.773024: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.773031: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.773417: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.773424: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.773810: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.773816: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.774202: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.774227: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.774613: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.774619: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [001] .....  4975.775371: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [00-00]
 veliad-hardware-8109    [001] .....  4975.775372: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [001] .....  4975.775373: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.775758: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.775761: smbus_read: i2c-2 a=058 f=0004 c=90 WORD_DATA
 veliad-hardware-8109    [001] .....  4975.776514: smbus_reply: i2c-2 a=058 f=0004 c=90 WORD_DATA l=2 [64-1a]
 veliad-hardware-8109    [001] .....  4975.776515: smbus_result: i2c-2 a=058 f=0004 c=90 WORD_DATA rd res=0
 veliad-hardware-8109    [001] .....  4975.776516: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [001] .....  4975.777295: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [00-00]
 veliad-hardware-8109    [001] .....  4975.777296: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [001] .....  4975.777298: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.777683: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.777685: smbus_read: i2c-2 a=058 f=0004 c=3a BYTE_DATA
 veliad-hardware-8109    [001] .....  4975.778317: smbus_reply: i2c-2 a=058 f=0004 c=3a BYTE_DATA l=1 [90]
 veliad-hardware-8109    [001] .....  4975.778318: smbus_result: i2c-2 a=058 f=0004 c=3a BYTE_DATA rd res=0
 veliad-hardware-8109    [001] .....  4975.778321: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [001] .....  4975.778708: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [001] .....  4975.778716: smbus_read: i2c-2 a=058 f=0004 c=81 BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.780196: smbus_reply: i2c-2 a=058 f=0004 c=81 BYTE_DATA l=1 [00]
 veliad-hardware-8109    [000] .....  4975.780198: smbus_result: i2c-2 a=058 f=0004 c=81 BYTE_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.780201: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [000] .....  4975.780953: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [00-00]
 veliad-hardware-8109    [000] .....  4975.780954: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.780956: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.781342: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.781348: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.781748: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.782450: smbus_write: i2c-2 a=058 f=0004 c=0 BYTE_DATA l=1 [00]
 veliad-hardware-8109    [000] .....  4975.782973: smbus_result: i2c-2 a=058 f=0004 c=0 BYTE_DATA wr res=0
 veliad-hardware-8109    [000] .....  4975.782975: smbus_read: i2c-2 a=058 f=0004 c=0 BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.783609: smbus_reply: i2c-2 a=058 f=0004 c=0 BYTE_DATA l=1 [00]
 veliad-hardware-8109    [000] .....  4975.783610: smbus_result: i2c-2 a=058 f=0004 c=0 BYTE_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.783611: smbus_read: i2c-2 a=058 f=0004 c=99 BLOCK_DATA
 veliad-hardware-8109    [000] .....  4975.785205: smbus_reply: i2c-2 a=058 f=0004 c=99 BLOCK_DATA l=9 [08-33-59-20-50-4f-57-45-52]
 veliad-hardware-8109    [000] .....  4975.785206: smbus_result: i2c-2 a=058 f=0004 c=99 BLOCK_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.785209: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [000] .....  4975.785960: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [00-00]
 veliad-hardware-8109    [000] .....  4975.785961: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.785963: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.790107: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.790127: smbus_read: i2c-2 a=058 f=0004 c=9a BLOCK_DATA
 veliad-hardware-8109    [000] .....  4975.792417: smbus_reply: i2c-2 a=058 f=0004 c=9a BLOCK_DATA l=9 [08-59-4d-2d-32-31-35-31-45]
 veliad-hardware-8109    [000] .....  4975.792420: smbus_result: i2c-2 a=058 f=0004 c=9a BLOCK_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.792423: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [000] .....  4975.793280: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [00-00]
 veliad-hardware-8109    [000] .....  4975.793281: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.793283: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.793669: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.793679: smbus_read: i2c-2 a=058 f=0004 c=9b BLOCK_DATA
 veliad-hardware-8109    [000] .....  4975.794479: smbus_reply: i2c-2 a=058 f=0004 c=9b BLOCK_DATA l=2 [01-41]
 veliad-hardware-8109    [000] .....  4975.794480: smbus_result: i2c-2 a=058 f=0004 c=9b BLOCK_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.794482: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [000] .....  4975.795233: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [00-00]
 veliad-hardware-8109    [000] .....  4975.795234: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.795236: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.795621: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.795628: smbus_read: i2c-2 a=058 f=0004 c=9c BLOCK_DATA
 veliad-hardware-8109    [000] .....  4975.802397: smbus_result: i2c-2 a=058 f=0004 c=9c BLOCK_DATA rd res=-71
 veliad-hardware-8109    [000] .....  4975.802399: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.802786: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.802788: smbus_read: i2c-2 a=058 f=0004 c=9d BLOCK_DATA
 veliad-hardware-8109    [000] .....  4975.809121: smbus_result: i2c-2 a=058 f=0004 c=9d BLOCK_DATA rd res=-71
 veliad-hardware-8109    [000] .....  4975.809124: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.809511: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.809514: smbus_read: i2c-2 a=058 f=0004 c=9e BLOCK_DATA
 veliad-hardware-8109    [000] .....  4975.816401: smbus_result: i2c-2 a=058 f=0004 c=9e BLOCK_DATA rd res=-71
 veliad-hardware-8109    [000] .....  4975.816406: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.816821: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.816851: smbus_read: i2c-2 a=058 f=0004 c=7e BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.817485: smbus_reply: i2c-2 a=058 f=0004 c=7e BYTE_DATA l=1 [00]
 veliad-hardware-8109    [000] .....  4975.817486: smbus_result: i2c-2 a=058 f=0004 c=7e BYTE_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.817488: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [000] .....  4975.818240: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [00-00]
 veliad-hardware-8109    [000] .....  4975.818240: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.818242: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.818628: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.818634: smbus_read: i2c-2 a=058 f=0004 c=7f BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.819267: smbus_result: i2c-2 a=058 f=0004 c=7f BYTE_DATA rd res=-74
 veliad-hardware-8109    [000] .....  4975.819268: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.819653: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.819666: smbus_read: i2c-2 a=058 f=0004 c=80 BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.820302: smbus_result: i2c-2 a=058 f=0004 c=80 BYTE_DATA rd res=-74
 veliad-hardware-8109    [000] .....  4975.820303: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.820688: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.820701: smbus_write: i2c-2 a=058 f=0004 c=0 BYTE_DATA l=1 [20]
 veliad-hardware-8109    [000] .....  4975.821216: smbus_result: i2c-2 a=058 f=0004 c=0 BYTE_DATA wr res=0
 veliad-hardware-8109    [000] .....  4975.876587: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [000] .....  4975.877347: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [00-00]
 veliad-hardware-8109    [000] .....  4975.877349: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.877352: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.877737: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.877750: smbus_read: i2c-2 a=058 f=0004 c=7e BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.878382: smbus_reply: i2c-2 a=058 f=0004 c=7e BYTE_DATA l=1 [00]
 veliad-hardware-8109    [000] .....  4975.878382: smbus_result: i2c-2 a=058 f=0004 c=7e BYTE_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.878384: smbus_read: i2c-2 a=058 f=0004 c=79 WORD_DATA
 veliad-hardware-8109    [000] .....  4975.879134: smbus_reply: i2c-2 a=058 f=0004 c=79 WORD_DATA l=2 [00-00]
 veliad-hardware-8109    [000] .....  4975.879135: smbus_result: i2c-2 a=058 f=0004 c=79 WORD_DATA rd res=0
 veliad-hardware-8109    [000] .....  4975.879136: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.879521: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.879527: smbus_read: i2c-2 a=058 f=0004 c=7f BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.880350: smbus_result: i2c-2 a=058 f=0004 c=7f BYTE_DATA rd res=-74
 veliad-hardware-8109    [000] .....  4975.880352: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.880738: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0
 veliad-hardware-8109    [000] .....  4975.880755: smbus_read: i2c-2 a=058 f=0004 c=80 BYTE_DATA
 veliad-hardware-8109    [000] .....  4975.881394: smbus_result: i2c-2 a=058 f=0004 c=80 BYTE_DATA rd res=-74
 veliad-hardware-8109    [000] .....  4975.881396: smbus_write: i2c-2 a=058 f=0004 c=3 BYTE l=0 []
 veliad-hardware-8109    [000] .....  4975.881781: smbus_result: i2c-2 a=058 f=0004 c=3 BYTE wr res=0

This is the kernel patch that was used:

diff --git a/drivers/hwmon/pmbus/fsp-3y.c b/drivers/hwmon/pmbus/fsp-3y.c
index a120fe05ffb2..b29e274b1acf 100644
--- a/drivers/hwmon/pmbus/fsp-3y.c
+++ b/drivers/hwmon/pmbus/fsp-3y.c
@@ -176,6 +176,7 @@ static struct pmbus_driver_info fsp3y_info[] = {
                        PMBUS_HAVE_PIN | PMBUS_HAVE_POUT  |
                        PMBUS_HAVE_TEMP | PMBUS_HAVE_TEMP2 |
                        PMBUS_HAVE_VIN | PMBUS_HAVE_IIN |
+                       PMBUS_HAVE_STATUS_VOUT | PMBUS_HAVE_STATUS_IOUT | PMBUS_HAVE_STATUS_INPUT | PMBUS_HAVE_STATUS_TEMP | PMBUS_HAVE_STATUS_FAN12 |
                        PMBUS_HAVE_FAN12,
                .func[YM2151_PAGE_5VSB_LOG] =
                        PMBUS_HAVE_VOUT | PMBUS_HAVE_IOUT,
diff --git a/drivers/hwmon/pmbus/pmbus.c b/drivers/hwmon/pmbus/pmbus.c
index 59424dc518c8..5fb6a91427dd 100644
--- a/drivers/hwmon/pmbus/pmbus.c
+++ b/drivers/hwmon/pmbus/pmbus.c
@@ -4,6 +4,7 @@
  *
  * Copyright (c) 2010, 2011 Ericsson AB.
  */
+#define DEBUG
 
 #include <linux/kernel.h>
 #include <linux/module.h>
diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c
index 019c5982ba56..9212d2e0bdd1 100644
--- a/drivers/hwmon/pmbus/pmbus_core.c
+++ b/drivers/hwmon/pmbus/pmbus_core.c
@@ -5,6 +5,7 @@
  * Copyright (c) 2010, 2011 Ericsson AB.
  * Copyright (c) 2012 Guenter Roeck
  */
+#define DEBUG
 
 #include <linux/debugfs.h>
 #include <linux/kernel.h>
@@ -544,8 +545,13 @@ static int pmbus_check_status_cml(struct i2c_client *client)
        status = data->read_status(client, -1);
        if (status < 0 || (status & PB_STATUS_CML)) {
                status2 = _pmbus_read_byte_data(client, -1, PMBUS_STATUS_CML);
-               if (status2 < 0 || (status2 & PB_CML_FAULT_INVALID_COMMAND))
+               if (status2 < 0 || (status2 & PB_CML_FAULT_INVALID_COMMAND)) {
+                       dev_dbg(&client->dev, "pmbus_check_status_cml: PB_CML_FAULT_INVALID_COMMAND in PMBUS_STATUS_CML: -EIO\n");
                        return -EIO;
+               } else {
+                       dev_dbg(&client->dev, "pmbus_check_status_cml: status=%x PB_STATUS_CML %d\n", status, (status & PB_STATUS_CML) ? 1 : 0);
+                       dev_dbg(&client->dev, "pmbus_check_status_cml: status2=%x PB_CML_FAULT_INVALID_COMMAND %d\n", status2,  (status2 & PB_CML_FAULT_INVALID_COMMAND) ? 1 : 0);
+               }
        }
        return 0;
 }
@@ -565,6 +571,10 @@ static bool pmbus_check_register(struct i2c_client *client,
                data->read_status(client, -1);
        if (reg < PMBUS_VIRT_BASE)
                pmbus_clear_fault_page(client, -1);
+
+       if (rv < 0) {
+               dev_dbg(&client->dev, "pmbus_check_register(reg 0x%x): failing due to rv 0x%x\n", reg, rv);
+       }
        return rv >= 0;
 }
 
@@ -577,11 +587,16 @@ static bool pmbus_check_status_register(struct i2c_client *client, int page)
        if (status >= 0 && !(data->flags & PMBUS_SKIP_STATUS_CHECK) &&
            (status & PB_STATUS_CML)) {
                status = _pmbus_read_byte_data(client, -1, PMBUS_STATUS_CML);
-               if (status < 0 || (status & PB_CML_FAULT_INVALID_COMMAND))
+               if (status < 0 || (status & PB_CML_FAULT_INVALID_COMMAND)) {
                        status = -EIO;
+                       dev_dbg(&client->dev, "pmbus_check_status_register(page %x): status 0x%x, PB_CML_FAULT_INVALID_COMMAND %d\n", page, status, (status & PB_CML_FAULT_INVALID_COMMAND) ? 1 : 0);
+               }
        }
 
        pmbus_clear_fault_page(client, -1);
+       if (status < 0) {
+               dev_dbg(&client->dev, "pmbus_check_status_register(page 0x%x): failing due to status 0x%x\n", page, status);
+       }
        return status >= 0;
 }
 
@@ -2473,6 +2488,7 @@ static int pmbus_read_coefficients(struct i2c_client *client,
        info->m[class] = m;
        info->b[class] = b;
        info->R[class] = R;
+       dev_dbg(&client->dev, "pmbus_read_coefficients: m=%hx, b=%hx, R=%hhx\n", m, b, R);
 
        return rv;
 }
@@ -2486,15 +2502,19 @@ static int pmbus_init_coefficients(struct i2c_client *client,
 
        for (i = 0; i < ARRAY_SIZE(class_attr_map); i++) {
                map = &class_attr_map[i];
+               dev_dbg(&client->dev, "pmbus_init_coefficients: i=%d, info->format[map->class]=%d\n", i, info->format[map->class]);
                if (info->format[map->class] != direct)
                        continue;
                for (n = 0; n < map->nattr; n++) {
                        attr = &map->attr[n];
+                       dev_dbg(&client->dev, "pmbus_init_coefficients: i=%d, n=%d: map->class=%d, attr->class=%d\n", i, n, map->class, attr->class);
                        if (map->class != attr->class)
                                continue;
                        ret = pmbus_read_coefficients(client, info, attr);
-                       if (ret >= 0)
+                       if (ret >= 0) {
+                               dev_err(&client->dev, "pmbus_read_coefficients failed %d\n", ret);
                                break;
+                       }
                }
                if (ret < 0) {
                        dev_err(&client->dev,
@@ -2516,9 +2536,14 @@ static int pmbus_identify_common(struct i2c_client *client,
 {
        int vout_mode = -1;
 
-       if (pmbus_check_byte_register(client, page, PMBUS_VOUT_MODE))
+       if (pmbus_check_byte_register(client, page, PMBUS_VOUT_MODE)) {
                vout_mode = _pmbus_read_byte_data(client, page,
                                                  PMBUS_VOUT_MODE);
+               dev_dbg(&client->dev, "VOUT_MODE at page %d: hex %x\n", page, vout_mode);
+       } else {
+               dev_dbg(&client->dev, "PMBUS_VOUT_MODE: pmbus_check_byte_register failed for page %d; using hex value of %x\n", page, vout_mode);
+       }
+
        if (vout_mode >= 0 && vout_mode != 0xff) {
                /*
                 * Not all chips support the VOUT_MODE command,
@@ -2530,22 +2555,28 @@ static int pmbus_identify_common(struct i2c_client *client,
                                return -ENODEV;
 
                        data->exponent[page] = ((s8)(vout_mode << 3)) >> 3;
+                       dev_dbg(&client->dev, "VOUT_MODE: linear, exponent=%hhx\n", ((s8)(vout_mode << 3)) >> 3);
                        break;
                case 1: /* VID mode         */
                        if (data->info->format[PSC_VOLTAGE_OUT] != vid)
                                return -ENODEV;
+                       dev_dbg(&client->dev, "VOUT_MODE: VID\n");
                        break;
                case 2: /* direct mode      */
                        if (data->info->format[PSC_VOLTAGE_OUT] != direct)
                                return -ENODEV;
+                       dev_dbg(&client->dev, "VOUT_MODE: direct\n");
                        break;
                case 3: /* ieee 754 half precision */
                        if (data->info->format[PSC_VOLTAGE_OUT] != ieee754)
                                return -ENODEV;
+                       dev_dbg(&client->dev, "VOUT_MODE: ieee754\n");
                        break;
                default:
                        return -ENODEV;
                }
+       } else {
+               dev_dbg(&client->dev, "VOUT_MODE was out of range, so we aren't doing a thing\n");
        }
 
        return 0;